{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/13, 10:36:23, Iteration: 500, Train Loss: 5.0974e+00, c1: 8.3738e-02, c2: 2.9128e-02, d1: 4.9102e-01, d2: 1.6013e-03, l1_reg: 3.0264e+02, l2_reg: 4.2575e+01\n",
      "2020/09/13, 10:36:28, Iteration: 1000, Train Loss: 5.1105e+00, c1: 1.0503e-01, c2: 1.7523e-02, d1: 4.9442e-01, d2: 8.6920e-04, l1_reg: 3.0123e+02, l2_reg: 4.2103e+01\n",
      "2020/09/13, 10:36:33, Iteration: 1500, Train Loss: 5.0901e+00, c1: 8.6108e-02, c2: 8.3820e-03, d1: 4.9721e-01, d2: 6.7030e-04, l1_reg: 2.9989e+02, l2_reg: 4.1771e+01\n",
      "2020/09/13, 10:36:38, Iteration: 2000, Train Loss: 5.0411e+00, c1: 1.9529e-02, c2: 1.2766e-02, d1: 4.9621e-01, d2: 2.1212e-03, l1_reg: 2.9938e+02, l2_reg: 4.1681e+01\n",
      "2020/09/13, 10:36:42, Iteration: 2500, Train Loss: 5.0262e+00, c1: 1.6549e-02, c2: 1.9742e-03, d1: 4.9807e-01, d2: 2.3009e-03, l1_reg: 2.9903e+02, l2_reg: 4.1661e+01\n",
      "2020/09/13, 10:36:47, Iteration: 3000, Train Loss: 5.0340e+00, c1: 3.2965e-02, c2: 4.6614e-03, d1: 4.9768e-01, d2: 1.0268e-03, l1_reg: 2.9905e+02, l2_reg: 4.1770e+01\n",
      "2020/09/13, 10:36:52, Iteration: 3500, Train Loss: 5.0320e+00, c1: 1.8092e-02, c2: 1.1726e-02, d1: 4.9582e-01, d2: 2.0544e-03, l1_reg: 2.9950e+02, l2_reg: 4.2045e+01\n",
      "2020/09/13, 10:36:56, Iteration: 4000, Train Loss: 4.9947e+00, c1: 4.9527e-03, c2: 7.2636e-03, d1: 4.9433e-01, d2: 2.4650e-03, l1_reg: 3.0105e+02, l2_reg: 4.2695e+01\n",
      "2020/09/13, 10:37:01, Iteration: 4500, Train Loss: 4.9211e+00, c1: 1.7296e-02, c2: 6.2091e-02, d1: 4.6638e-01, d2: 5.3652e-03, l1_reg: 3.0675e+02, l2_reg: 4.5196e+01\n",
      "2020/09/13, 10:37:06, Iteration: 5000, Train Loss: 2.4001e+00, c1: 2.1902e-01, c2: 4.4899e-01, d1: 6.5336e-02, d2: 1.8072e-02, l1_reg: 3.2875e+02, l2_reg: 5.6737e+01\n",
      "2020/09/13, 10:37:10, Iteration: 5500, Train Loss: 2.0833e+00, c1: 2.0597e-01, c2: 3.7725e-01, d1: 6.2552e-02, d2: 1.2001e-02, l1_reg: 3.3312e+02, l2_reg: 5.9065e+01\n",
      "2020/09/13, 10:37:15, Iteration: 6000, Train Loss: 1.9031e+00, c1: 1.3928e-01, c2: 3.2771e-01, d1: 5.8814e-02, d2: 1.9250e-02, l1_reg: 3.3818e+02, l2_reg: 6.1436e+01\n",
      "2020/09/13, 10:37:20, Iteration: 6500, Train Loss: 1.7036e+00, c1: 6.8695e-02, c2: 3.1447e-01, d1: 5.7473e-02, d2: 1.1681e-02, l1_reg: 3.4094e+02, l2_reg: 6.2750e+01\n",
      "2020/09/13, 10:37:24, Iteration: 7000, Train Loss: 1.6712e+00, c1: 7.7618e-02, c2: 3.0789e-01, d1: 5.6530e-02, d2: 1.0462e-02, l1_reg: 3.4273e+02, l2_reg: 6.3643e+01\n",
      "2020/09/13, 10:37:29, Iteration: 7500, Train Loss: 1.7227e+00, c1: 9.1540e-02, c2: 2.9994e-01, d1: 5.6654e-02, d2: 1.6485e-02, l1_reg: 3.4414e+02, l2_reg: 6.4373e+01\n",
      "2020/09/13, 10:37:34, Iteration: 8000, Train Loss: 1.8605e+00, c1: 1.4734e-01, c2: 2.9799e-01, d1: 5.5738e-02, d2: 2.6182e-02, l1_reg: 3.4572e+02, l2_reg: 6.5160e+01\n",
      "2020/09/13, 10:37:39, Iteration: 8500, Train Loss: 1.7344e+00, c1: 1.3662e-01, c2: 2.8222e-01, d1: 5.5538e-02, d2: 1.9568e-02, l1_reg: 3.4761e+02, l2_reg: 6.6070e+01\n",
      "2020/09/13, 10:37:44, Iteration: 9000, Train Loss: 1.6118e+00, c1: 7.4674e-02, c2: 2.7176e-01, d1: 5.4872e-02, d2: 1.7315e-02, l1_reg: 3.5007e+02, l2_reg: 6.7238e+01\n",
      "2020/09/13, 10:37:48, Iteration: 9500, Train Loss: 1.4881e+00, c1: 1.2219e-01, c2: 2.5119e-01, d1: 5.4549e-02, d2: 6.6896e-03, l1_reg: 3.5300e+02, l2_reg: 6.8662e+01\n",
      "2020/09/13, 10:37:53, Iteration: 10000, Train Loss: 1.3599e+00, c1: 7.6080e-02, c2: 2.2121e-01, d1: 5.4132e-02, d2: 7.8865e-03, l1_reg: 3.5651e+02, l2_reg: 7.0370e+01\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3.8/site-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  return array(a, dtype, copy=False, order=order)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/13, 10:37:58, Iteration: 10500, Train Loss: 1.3774e+00, c1: 1.8204e-01, c2: 1.9419e-01, d1: 5.2321e-02, d2: 8.9623e-03, l1_reg: 3.6021e+02, l2_reg: 7.2188e+01\n",
      "2020/09/13, 10:38:03, Iteration: 11000, Train Loss: 1.2840e+00, c1: 1.2152e-01, c2: 1.7557e-01, d1: 5.0319e-02, d2: 1.3257e-02, l1_reg: 3.6337e+02, l2_reg: 7.3738e+01\n",
      "2020/09/13, 10:38:07, Iteration: 11500, Train Loss: 1.1092e+00, c1: 8.5394e-02, c2: 1.5893e-01, d1: 4.7683e-02, d2: 7.0168e-03, l1_reg: 3.6618e+02, l2_reg: 7.5128e+01\n",
      "2020/09/13, 10:38:12, Iteration: 12000, Train Loss: 1.1603e+00, c1: 1.2862e-01, c2: 1.5011e-01, d1: 4.4728e-02, d2: 1.3410e-02, l1_reg: 3.6902e+02, l2_reg: 7.6556e+01\n",
      "2020/09/13, 10:38:17, Iteration: 12500, Train Loss: 9.6597e-01, c1: 5.5205e-02, c2: 1.3790e-01, d1: 4.2148e-02, d2: 7.5590e-03, l1_reg: 3.7187e+02, l2_reg: 7.8021e+01\n",
      "2020/09/13, 10:38:21, Iteration: 13000, Train Loss: 9.0928e-01, c1: 6.3109e-02, c2: 1.2837e-01, d1: 3.9370e-02, d2: 6.7352e-03, l1_reg: 3.7486e+02, l2_reg: 7.9564e+01\n",
      "2020/09/13, 10:38:26, Iteration: 13500, Train Loss: 8.3376e-01, c1: 7.5751e-02, c2: 1.2028e-01, d1: 3.6579e-02, d2: 3.1372e-03, l1_reg: 3.7761e+02, l2_reg: 8.0920e+01\n",
      "2020/09/13, 10:38:31, Iteration: 14000, Train Loss: 7.7410e-01, c1: 5.9714e-02, c2: 1.1159e-01, d1: 3.4888e-02, d2: 3.0723e-03, l1_reg: 3.8000e+02, l2_reg: 8.2103e+01\n",
      "2020/09/13, 10:38:35, Iteration: 14500, Train Loss: 7.7058e-01, c1: 5.2178e-02, c2: 1.1012e-01, d1: 3.3164e-02, d2: 5.6405e-03, l1_reg: 3.8184e+02, l2_reg: 8.2998e+01\n",
      "2020/09/13, 10:38:40, Iteration: 15000, Train Loss: 7.9627e-01, c1: 1.0340e-01, c2: 1.0703e-01, d1: 3.2115e-02, d2: 5.0624e-03, l1_reg: 3.8331e+02, l2_reg: 8.3721e+01\n",
      "2020/09/13, 10:38:45, Iteration: 15500, Train Loss: 7.7777e-01, c1: 5.4769e-02, c2: 1.0508e-01, d1: 3.0837e-02, d2: 9.9390e-03, l1_reg: 3.8461e+02, l2_reg: 8.4380e+01\n",
      "2020/09/13, 10:38:50, Iteration: 16000, Train Loss: 8.0958e-01, c1: 6.9966e-02, c2: 1.0582e-01, d1: 3.0800e-02, d2: 1.1414e-02, l1_reg: 3.8580e+02, l2_reg: 8.4981e+01\n",
      "2020/09/13, 10:38:54, Iteration: 16500, Train Loss: 7.8654e-01, c1: 9.8765e-02, c2: 1.0341e-01, d1: 2.9854e-02, d2: 7.8992e-03, l1_reg: 3.8701e+02, l2_reg: 8.5615e+01\n",
      "2020/09/13, 10:38:59, Iteration: 17000, Train Loss: 7.6566e-01, c1: 8.1602e-02, c2: 1.0204e-01, d1: 2.9155e-02, d2: 8.6381e-03, l1_reg: 3.8829e+02, l2_reg: 8.6293e+01\n",
      "2020/09/13, 10:39:04, Iteration: 17500, Train Loss: 6.8817e-01, c1: 6.1012e-02, c2: 9.9163e-02, d1: 2.8629e-02, d2: 4.3380e-03, l1_reg: 3.8950e+02, l2_reg: 8.6941e+01\n",
      "2020/09/13, 10:39:09, Iteration: 18000, Train Loss: 7.1521e-01, c1: 6.4541e-02, c2: 9.6576e-02, d1: 2.9121e-02, d2: 6.9734e-03, l1_reg: 3.9074e+02, l2_reg: 8.7603e+01\n",
      "2020/09/13, 10:39:14, Iteration: 18500, Train Loss: 6.9536e-01, c1: 8.0407e-02, c2: 9.8085e-02, d1: 2.8200e-02, d2: 3.8696e-03, l1_reg: 3.9202e+02, l2_reg: 8.8295e+01\n",
      "2020/09/13, 10:39:18, Iteration: 19000, Train Loss: 7.2954e-01, c1: 6.2432e-02, c2: 9.9033e-02, d1: 2.6876e-02, d2: 1.0124e-02, l1_reg: 3.9333e+02, l2_reg: 8.9004e+01\n",
      "2020/09/13, 10:39:23, Iteration: 19500, Train Loss: 7.3697e-01, c1: 1.0186e-01, c2: 9.3647e-02, d1: 2.6467e-02, d2: 8.9498e-03, l1_reg: 3.9453e+02, l2_reg: 8.9679e+01\n",
      "2020/09/13, 10:39:28, Iteration: 20000, Train Loss: 7.0875e-01, c1: 7.1484e-02, c2: 9.0167e-02, d1: 2.6663e-02, d2: 1.0013e-02, l1_reg: 3.9583e+02, l2_reg: 9.0383e+01\n",
      "2020/09/13, 10:39:33, Iteration: 20500, Train Loss: 7.5822e-01, c1: 6.1950e-02, c2: 8.3203e-02, d1: 2.5757e-02, d2: 1.8909e-02, l1_reg: 3.9705e+02, l2_reg: 9.1085e+01\n",
      "2020/09/13, 10:39:37, Iteration: 21000, Train Loss: 7.7143e-01, c1: 9.8100e-02, c2: 8.5126e-02, d1: 2.4881e-02, d2: 1.6913e-02, l1_reg: 3.9838e+02, l2_reg: 9.1831e+01\n",
      "2020/09/13, 10:39:42, Iteration: 21500, Train Loss: 6.8339e-01, c1: 1.0677e-01, c2: 8.4113e-02, d1: 2.5294e-02, d2: 7.1333e-03, l1_reg: 3.9969e+02, l2_reg: 9.2559e+01\n",
      "2020/09/13, 10:39:47, Iteration: 22000, Train Loss: 6.9403e-01, c1: 1.1383e-01, c2: 8.0971e-02, d1: 2.4798e-02, d2: 8.9303e-03, l1_reg: 4.0094e+02, l2_reg: 9.3274e+01\n",
      "2020/09/13, 10:39:51, Iteration: 22500, Train Loss: 6.7148e-01, c1: 4.1754e-02, c2: 8.0245e-02, d1: 2.3973e-02, d2: 1.4926e-02, l1_reg: 4.0213e+02, l2_reg: 9.3954e+01\n",
      "2020/09/13, 10:39:56, Iteration: 23000, Train Loss: 6.4502e-01, c1: 7.0959e-02, c2: 7.7235e-02, d1: 2.3832e-02, d2: 1.0403e-02, l1_reg: 4.0330e+02, l2_reg: 9.4627e+01\n",
      "2020/09/13, 10:40:01, Iteration: 23500, Train Loss: 7.2481e-01, c1: 7.8642e-02, c2: 8.2175e-02, d1: 2.4700e-02, d2: 1.5264e-02, l1_reg: 4.0438e+02, l2_reg: 9.5263e+01\n",
      "2020/09/13, 10:40:06, Iteration: 24000, Train Loss: 6.2990e-01, c1: 6.5655e-02, c2: 7.6829e-02, d1: 2.3371e-02, d2: 1.0004e-02, l1_reg: 4.0534e+02, l2_reg: 9.5844e+01\n",
      "2020/09/13, 10:40:11, Iteration: 24500, Train Loss: 7.0626e-01, c1: 8.8153e-02, c2: 7.5671e-02, d1: 2.3738e-02, d2: 1.5371e-02, l1_reg: 4.0621e+02, l2_reg: 9.6377e+01\n",
      "2020/09/13, 10:40:15, Iteration: 25000, Train Loss: 6.2187e-01, c1: 6.5566e-02, c2: 7.1446e-02, d1: 2.2830e-02, d2: 1.1367e-02, l1_reg: 4.0698e+02, l2_reg: 9.6856e+01\n",
      "2020/09/13, 10:40:20, Iteration: 25500, Train Loss: 5.9578e-01, c1: 7.0642e-02, c2: 7.6194e-02, d1: 2.2679e-02, d2: 6.9763e-03, l1_reg: 4.0777e+02, l2_reg: 9.7342e+01\n",
      "2020/09/13, 10:40:25, Iteration: 26000, Train Loss: 6.2560e-01, c1: 7.3023e-02, c2: 7.1465e-02, d1: 2.2420e-02, d2: 1.1398e-02, l1_reg: 4.0850e+02, l2_reg: 9.7811e+01\n",
      "2020/09/13, 10:40:30, Iteration: 26500, Train Loss: 6.2817e-01, c1: 8.1566e-02, c2: 7.1061e-02, d1: 2.3452e-02, d2: 9.8892e-03, l1_reg: 4.0919e+02, l2_reg: 9.8250e+01\n",
      "2020/09/13, 10:40:34, Iteration: 27000, Train Loss: 5.7021e-01, c1: 7.6992e-02, c2: 7.2553e-02, d1: 2.3038e-02, d2: 4.5172e-03, l1_reg: 4.0986e+02, l2_reg: 9.8686e+01\n",
      "2020/09/13, 10:40:39, Iteration: 27500, Train Loss: 6.6977e-01, c1: 8.4057e-02, c2: 7.1125e-02, d1: 2.2299e-02, d2: 1.4934e-02, l1_reg: 4.1051e+02, l2_reg: 9.9100e+01\n",
      "2020/09/13, 10:40:44, Iteration: 28000, Train Loss: 5.9625e-01, c1: 8.1500e-02, c2: 6.7490e-02, d1: 2.1452e-02, d2: 9.7758e-03, l1_reg: 4.1108e+02, l2_reg: 9.9481e+01\n",
      "2020/09/13, 10:40:48, Iteration: 28500, Train Loss: 6.1550e-01, c1: 7.8026e-02, c2: 6.7497e-02, d1: 2.1586e-02, d2: 1.1912e-02, l1_reg: 4.1168e+02, l2_reg: 9.9866e+01\n",
      "2020/09/13, 10:40:53, Iteration: 29000, Train Loss: 5.2925e-01, c1: 6.5261e-02, c2: 6.9399e-02, d1: 2.0850e-02, d2: 4.7290e-03, l1_reg: 4.1225e+02, l2_reg: 1.0024e+02\n",
      "2020/09/13, 10:40:58, Iteration: 29500, Train Loss: 5.1427e-01, c1: 5.6101e-02, c2: 6.5660e-02, d1: 2.0912e-02, d2: 5.2058e-03, l1_reg: 4.1270e+02, l2_reg: 1.0055e+02\n",
      "2020/09/13, 10:41:03, Iteration: 30000, Train Loss: 5.6968e-01, c1: 7.3665e-02, c2: 6.8695e-02, d1: 2.1028e-02, d2: 7.9643e-03, l1_reg: 4.1320e+02, l2_reg: 1.0090e+02\n",
      "2020/09/13, 10:41:07, Iteration: 30500, Train Loss: 5.7963e-01, c1: 5.8057e-02, c2: 7.0107e-02, d1: 2.0760e-02, d2: 1.0365e-02, l1_reg: 4.1374e+02, l2_reg: 1.0123e+02\n",
      "2020/09/13, 10:41:12, Iteration: 31000, Train Loss: 6.2423e-01, c1: 7.0462e-02, c2: 7.3355e-02, d1: 2.2205e-02, d2: 1.1164e-02, l1_reg: 4.1420e+02, l2_reg: 1.0156e+02\n",
      "2020/09/13, 10:41:17, Iteration: 31500, Train Loss: 5.4726e-01, c1: 8.8522e-02, c2: 6.4322e-02, d1: 2.0183e-02, d2: 6.3932e-03, l1_reg: 4.1471e+02, l2_reg: 1.0189e+02\n",
      "2020/09/13, 10:41:22, Iteration: 32000, Train Loss: 4.6305e-01, c1: 5.5822e-02, c2: 6.3079e-02, d1: 2.0086e-02, d2: 1.7127e-03, l1_reg: 4.1516e+02, l2_reg: 1.0221e+02\n",
      "2020/09/13, 10:41:26, Iteration: 32500, Train Loss: 5.0240e-01, c1: 4.4029e-02, c2: 6.4419e-02, d1: 2.0633e-02, d2: 5.8779e-03, l1_reg: 4.1567e+02, l2_reg: 1.0254e+02\n",
      "2020/09/13, 10:41:31, Iteration: 33000, Train Loss: 5.0673e-01, c1: 7.1202e-02, c2: 6.0039e-02, d1: 2.0567e-02, d2: 4.9745e-03, l1_reg: 4.1610e+02, l2_reg: 1.0284e+02\n",
      "2020/09/13, 10:41:36, Iteration: 33500, Train Loss: 5.5036e-01, c1: 5.6454e-02, c2: 6.3867e-02, d1: 2.0581e-02, d2: 9.6486e-03, l1_reg: 4.1650e+02, l2_reg: 1.0310e+02\n",
      "2020/09/13, 10:41:41, Iteration: 34000, Train Loss: 5.4571e-01, c1: 7.1046e-02, c2: 5.9485e-02, d1: 1.9933e-02, d2: 9.6871e-03, l1_reg: 4.1690e+02, l2_reg: 1.0342e+02\n",
      "2020/09/13, 10:41:46, Iteration: 34500, Train Loss: 5.5922e-01, c1: 5.2172e-02, c2: 6.8102e-02, d1: 2.0707e-02, d2: 9.5669e-03, l1_reg: 4.1739e+02, l2_reg: 1.0374e+02\n",
      "2020/09/13, 10:41:50, Iteration: 35000, Train Loss: 5.3215e-01, c1: 6.6782e-02, c2: 6.3184e-02, d1: 1.9765e-02, d2: 7.8164e-03, l1_reg: 4.1784e+02, l2_reg: 1.0404e+02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/13, 10:41:55, Iteration: 35500, Train Loss: 5.5732e-01, c1: 6.0889e-02, c2: 5.9455e-02, d1: 1.9633e-02, d2: 1.2173e-02, l1_reg: 4.1832e+02, l2_reg: 1.0438e+02\n",
      "2020/09/13, 10:41:59, Iteration: 36000, Train Loss: 5.0945e-01, c1: 7.8444e-02, c2: 6.0733e-02, d1: 1.9244e-02, d2: 5.6360e-03, l1_reg: 4.1879e+02, l2_reg: 1.0470e+02\n",
      "2020/09/13, 10:42:04, Iteration: 36500, Train Loss: 5.3275e-01, c1: 5.4718e-02, c2: 6.5388e-02, d1: 1.9915e-02, d2: 8.2707e-03, l1_reg: 4.1925e+02, l2_reg: 1.0501e+02\n",
      "2020/09/13, 10:42:09, Iteration: 37000, Train Loss: 5.2722e-01, c1: 6.0797e-02, c2: 5.7331e-02, d1: 1.9468e-02, d2: 9.9744e-03, l1_reg: 4.1970e+02, l2_reg: 1.0533e+02\n",
      "2020/09/13, 10:42:13, Iteration: 37500, Train Loss: 5.5752e-01, c1: 7.6308e-02, c2: 5.9250e-02, d1: 1.9196e-02, d2: 1.1150e-02, l1_reg: 4.2016e+02, l2_reg: 1.0564e+02\n",
      "2020/09/13, 10:42:18, Iteration: 38000, Train Loss: 5.6928e-01, c1: 6.0998e-02, c2: 6.2449e-02, d1: 2.0249e-02, d2: 1.1844e-02, l1_reg: 4.2056e+02, l2_reg: 1.0592e+02\n",
      "2020/09/13, 10:42:23, Iteration: 38500, Train Loss: 5.1382e-01, c1: 6.0586e-02, c2: 5.7847e-02, d1: 1.9379e-02, d2: 8.5900e-03, l1_reg: 4.2096e+02, l2_reg: 1.0620e+02\n",
      "2020/09/13, 10:42:27, Iteration: 39000, Train Loss: 4.7859e-01, c1: 6.6631e-02, c2: 5.6739e-02, d1: 1.9258e-02, d2: 4.9159e-03, l1_reg: 4.2138e+02, l2_reg: 1.0650e+02\n",
      "2020/09/13, 10:42:32, Iteration: 39500, Train Loss: 6.1456e-01, c1: 6.3729e-02, c2: 6.0611e-02, d1: 1.9059e-02, d2: 1.7840e-02, l1_reg: 4.2178e+02, l2_reg: 1.0679e+02\n",
      "2020/09/13, 10:42:37, Iteration: 40000, Train Loss: 4.7774e-01, c1: 3.9358e-02, c2: 5.7205e-02, d1: 1.9237e-02, d2: 7.4387e-03, l1_reg: 4.2216e+02, l2_reg: 1.0707e+02\n",
      "2020/09/13, 10:42:41, Iteration: 40500, Train Loss: 5.0284e-01, c1: 5.4790e-02, c2: 5.8351e-02, d1: 1.9921e-02, d2: 7.3785e-03, l1_reg: 4.2256e+02, l2_reg: 1.0735e+02\n",
      "2020/09/13, 10:42:46, Iteration: 41000, Train Loss: 4.7990e-01, c1: 7.1758e-02, c2: 5.8631e-02, d1: 1.8950e-02, d2: 4.2738e-03, l1_reg: 4.2303e+02, l2_reg: 1.0766e+02\n",
      "2020/09/13, 10:42:51, Iteration: 41500, Train Loss: 5.1477e-01, c1: 8.8553e-02, c2: 6.1323e-02, d1: 1.8671e-02, d2: 5.5536e-03, l1_reg: 4.2345e+02, l2_reg: 1.0795e+02\n",
      "2020/09/13, 10:42:56, Iteration: 42000, Train Loss: 5.2195e-01, c1: 6.6546e-02, c2: 5.6368e-02, d1: 1.8671e-02, d2: 9.9585e-03, l1_reg: 4.2392e+02, l2_reg: 1.0827e+02\n",
      "2020/09/13, 10:43:00, Iteration: 42500, Train Loss: 5.0917e-01, c1: 5.2996e-02, c2: 5.5612e-02, d1: 1.8543e-02, d2: 1.0390e-02, l1_reg: 4.2427e+02, l2_reg: 1.0852e+02\n",
      "2020/09/13, 10:43:05, Iteration: 43000, Train Loss: 4.5219e-01, c1: 4.8436e-02, c2: 6.0259e-02, d1: 1.8201e-02, d2: 4.0961e-03, l1_reg: 4.2467e+02, l2_reg: 1.0878e+02\n",
      "2020/09/13, 10:43:10, Iteration: 43500, Train Loss: 4.8502e-01, c1: 5.5329e-02, c2: 5.3124e-02, d1: 1.8116e-02, d2: 8.9150e-03, l1_reg: 4.2509e+02, l2_reg: 1.0906e+02\n",
      "2020/09/13, 10:43:15, Iteration: 44000, Train Loss: 4.4574e-01, c1: 6.6785e-02, c2: 5.4133e-02, d1: 1.8108e-02, d2: 3.5474e-03, l1_reg: 4.2549e+02, l2_reg: 1.0933e+02\n",
      "2020/09/13, 10:43:19, Iteration: 44500, Train Loss: 4.2675e-01, c1: 5.2387e-02, c2: 5.4505e-02, d1: 1.7952e-02, d2: 3.1328e-03, l1_reg: 4.2587e+02, l2_reg: 1.0962e+02\n",
      "2020/09/13, 10:43:24, Iteration: 45000, Train Loss: 4.3958e-01, c1: 4.2150e-02, c2: 5.3568e-02, d1: 1.7899e-02, d2: 5.7728e-03, l1_reg: 4.2623e+02, l2_reg: 1.0988e+02\n",
      "2020/09/13, 10:43:29, Iteration: 45500, Train Loss: 4.7105e-01, c1: 4.7040e-02, c2: 5.3908e-02, d1: 1.8595e-02, d2: 7.6332e-03, l1_reg: 4.2660e+02, l2_reg: 1.1013e+02\n",
      "2020/09/13, 10:43:33, Iteration: 46000, Train Loss: 4.2293e-01, c1: 5.0539e-02, c2: 5.5900e-02, d1: 1.8073e-02, d2: 2.3960e-03, l1_reg: 4.2698e+02, l2_reg: 1.1039e+02\n",
      "2020/09/13, 10:43:38, Iteration: 46500, Train Loss: 4.4258e-01, c1: 5.2669e-02, c2: 5.2279e-02, d1: 1.7783e-02, d2: 5.5242e-03, l1_reg: 4.2734e+02, l2_reg: 1.1066e+02\n",
      "2020/09/13, 10:43:43, Iteration: 47000, Train Loss: 5.5847e-01, c1: 7.2014e-02, c2: 5.8376e-02, d1: 1.8044e-02, d2: 1.3089e-02, l1_reg: 4.2772e+02, l2_reg: 1.1093e+02\n",
      "2020/09/13, 10:43:48, Iteration: 47500, Train Loss: 5.1013e-01, c1: 5.3799e-02, c2: 5.1992e-02, d1: 1.7544e-02, d2: 1.2491e-02, l1_reg: 4.2810e+02, l2_reg: 1.1120e+02\n",
      "2020/09/13, 10:43:52, Iteration: 48000, Train Loss: 4.6122e-01, c1: 4.9211e-02, c2: 5.6209e-02, d1: 1.7809e-02, d2: 6.5292e-03, l1_reg: 4.2845e+02, l2_reg: 1.1145e+02\n",
      "2020/09/13, 10:43:57, Iteration: 48500, Train Loss: 4.5971e-01, c1: 6.1649e-02, c2: 5.2704e-02, d1: 1.7402e-02, d2: 6.5925e-03, l1_reg: 4.2881e+02, l2_reg: 1.1170e+02\n",
      "2020/09/13, 10:44:02, Iteration: 49000, Train Loss: 5.0198e-01, c1: 6.1747e-02, c2: 5.4106e-02, d1: 1.7400e-02, d2: 1.0391e-02, l1_reg: 4.2913e+02, l2_reg: 1.1196e+02\n",
      "2020/09/13, 10:44:06, Iteration: 49500, Train Loss: 5.1155e-01, c1: 6.3914e-02, c2: 5.1181e-02, d1: 1.8279e-02, d2: 1.1131e-02, l1_reg: 4.2944e+02, l2_reg: 1.1221e+02\n",
      "2020/09/13, 10:44:11, Iteration: 50000, Train Loss: 4.7785e-01, c1: 4.7152e-02, c2: 4.9541e-02, d1: 1.7570e-02, d2: 1.0637e-02, l1_reg: 4.2979e+02, l2_reg: 1.1247e+02\n",
      "2020/09/13, 10:44:16, Iteration: 50500, Train Loss: 4.0517e-01, c1: 2.5941e-02, c2: 5.2183e-02, d1: 1.7592e-02, d2: 4.6746e-03, l1_reg: 4.3017e+02, l2_reg: 1.1272e+02\n",
      "2020/09/13, 10:44:21, Iteration: 51000, Train Loss: 4.3756e-01, c1: 5.5606e-02, c2: 5.2753e-02, d1: 1.7847e-02, d2: 4.5223e-03, l1_reg: 4.3050e+02, l2_reg: 1.1298e+02\n",
      "2020/09/13, 10:44:25, Iteration: 51500, Train Loss: 5.6036e-01, c1: 6.5058e-02, c2: 5.2565e-02, d1: 1.7358e-02, d2: 1.6402e-02, l1_reg: 4.3090e+02, l2_reg: 1.1326e+02\n",
      "2020/09/13, 10:44:30, Iteration: 52000, Train Loss: 6.1837e-01, c1: 6.5298e-02, c2: 5.8108e-02, d1: 1.7209e-02, d2: 2.0664e-02, l1_reg: 4.3127e+02, l2_reg: 1.1352e+02\n",
      "2020/09/13, 10:44:35, Iteration: 52500, Train Loss: 4.2955e-01, c1: 5.0629e-02, c2: 5.3935e-02, d1: 1.7136e-02, d2: 4.5745e-03, l1_reg: 4.3158e+02, l2_reg: 1.1377e+02\n",
      "2020/09/13, 10:44:39, Iteration: 53000, Train Loss: 4.9794e-01, c1: 4.9994e-02, c2: 5.0342e-02, d1: 1.7263e-02, d2: 1.2428e-02, l1_reg: 4.3194e+02, l2_reg: 1.1403e+02\n",
      "2020/09/13, 10:44:44, Iteration: 53500, Train Loss: 4.6702e-01, c1: 6.6356e-02, c2: 5.3302e-02, d1: 1.7019e-02, d2: 7.0558e-03, l1_reg: 4.3228e+02, l2_reg: 1.1428e+02\n",
      "2020/09/13, 10:44:49, Iteration: 54000, Train Loss: 5.0168e-01, c1: 5.2675e-02, c2: 5.4115e-02, d1: 1.8107e-02, d2: 1.0558e-02, l1_reg: 4.3268e+02, l2_reg: 1.1455e+02\n",
      "2020/09/13, 10:44:54, Iteration: 54500, Train Loss: 4.7867e-01, c1: 5.9873e-02, c2: 5.2991e-02, d1: 1.7229e-02, d2: 8.7525e-03, l1_reg: 4.3302e+02, l2_reg: 1.1482e+02\n",
      "2020/09/13, 10:44:58, Iteration: 55000, Train Loss: 4.1788e-01, c1: 4.4170e-02, c2: 4.9412e-02, d1: 1.7799e-02, d2: 4.7484e-03, l1_reg: 4.3338e+02, l2_reg: 1.1508e+02\n",
      "2020/09/13, 10:45:03, Iteration: 55500, Train Loss: 4.0026e-01, c1: 6.3493e-02, c2: 4.8065e-02, d1: 1.6630e-02, d2: 2.6270e-03, l1_reg: 4.3378e+02, l2_reg: 1.1536e+02\n",
      "2020/09/13, 10:45:08, Iteration: 56000, Train Loss: 5.1912e-01, c1: 5.4025e-02, c2: 5.6593e-02, d1: 1.6525e-02, d2: 1.3006e-02, l1_reg: 4.3413e+02, l2_reg: 1.1561e+02\n",
      "2020/09/13, 10:45:13, Iteration: 56500, Train Loss: 5.0050e-01, c1: 6.5230e-02, c2: 5.5581e-02, d1: 1.6790e-02, d2: 1.0062e-02, l1_reg: 4.3448e+02, l2_reg: 1.1585e+02\n",
      "2020/09/13, 10:45:17, Iteration: 57000, Train Loss: 4.1696e-01, c1: 7.0618e-02, c2: 5.1872e-02, d1: 1.6571e-02, d2: 2.5017e-03, l1_reg: 4.3481e+02, l2_reg: 1.1612e+02\n",
      "2020/09/13, 10:45:22, Iteration: 57500, Train Loss: 4.4464e-01, c1: 5.8285e-02, c2: 5.2690e-02, d1: 1.6711e-02, d2: 6.1168e-03, l1_reg: 4.3518e+02, l2_reg: 1.1638e+02\n",
      "2020/09/13, 10:45:27, Iteration: 58000, Train Loss: 4.7578e-01, c1: 5.9915e-02, c2: 5.2157e-02, d1: 1.7041e-02, d2: 8.8974e-03, l1_reg: 4.3554e+02, l2_reg: 1.1664e+02\n",
      "2020/09/13, 10:45:31, Iteration: 58500, Train Loss: 4.7693e-01, c1: 6.3036e-02, c2: 5.5429e-02, d1: 1.7240e-02, d2: 7.5198e-03, l1_reg: 4.3590e+02, l2_reg: 1.1692e+02\n",
      "2020/09/13, 10:45:36, Iteration: 59000, Train Loss: 4.8820e-01, c1: 4.6004e-02, c2: 5.5591e-02, d1: 1.6483e-02, d2: 1.1059e-02, l1_reg: 4.3622e+02, l2_reg: 1.1717e+02\n",
      "2020/09/13, 10:45:41, Iteration: 59500, Train Loss: 4.8734e-01, c1: 5.2322e-02, c2: 5.0654e-02, d1: 1.6461e-02, d2: 1.1844e-02, l1_reg: 4.3646e+02, l2_reg: 1.1738e+02\n",
      "2020/09/13, 10:45:46, Iteration: 60000, Train Loss: 5.5365e-01, c1: 5.0902e-02, c2: 5.5880e-02, d1: 1.6390e-02, d2: 1.7119e-02, l1_reg: 4.3677e+02, l2_reg: 1.1762e+02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/13, 10:45:50, Iteration: 60500, Train Loss: 4.1896e-01, c1: 5.3617e-02, c2: 4.7907e-02, d1: 1.6920e-02, d2: 5.2425e-03, l1_reg: 4.3701e+02, l2_reg: 1.1786e+02\n",
      "2020/09/13, 10:45:55, Iteration: 61000, Train Loss: 4.6342e-01, c1: 6.9964e-02, c2: 4.8872e-02, d1: 1.6791e-02, d2: 7.8922e-03, l1_reg: 4.3726e+02, l2_reg: 1.1808e+02\n",
      "2020/09/13, 10:46:00, Iteration: 61500, Train Loss: 4.8008e-01, c1: 5.8616e-02, c2: 4.7599e-02, d1: 1.6700e-02, d2: 1.1166e-02, l1_reg: 4.3750e+02, l2_reg: 1.1831e+02\n",
      "2020/09/13, 10:46:05, Iteration: 62000, Train Loss: 4.5308e-01, c1: 3.6956e-02, c2: 5.0323e-02, d1: 1.6075e-02, d2: 1.0441e-02, l1_reg: 4.3780e+02, l2_reg: 1.1856e+02\n",
      "2020/09/13, 10:46:09, Iteration: 62500, Train Loss: 4.8833e-01, c1: 4.9297e-02, c2: 4.8151e-02, d1: 1.6494e-02, d2: 1.2963e-02, l1_reg: 4.3811e+02, l2_reg: 1.1878e+02\n",
      "2020/09/13, 10:46:14, Iteration: 63000, Train Loss: 4.1412e-01, c1: 6.0427e-02, c2: 4.8547e-02, d1: 1.5939e-02, d2: 4.8657e-03, l1_reg: 4.3841e+02, l2_reg: 1.1903e+02\n",
      "2020/09/13, 10:46:19, Iteration: 63500, Train Loss: 4.5898e-01, c1: 5.3818e-02, c2: 5.0470e-02, d1: 1.6489e-02, d2: 8.8861e-03, l1_reg: 4.3872e+02, l2_reg: 1.1926e+02\n",
      "2020/09/13, 10:46:24, Iteration: 64000, Train Loss: 4.6744e-01, c1: 9.7692e-02, c2: 5.3517e-02, d1: 1.6444e-02, d2: 4.4753e-03, l1_reg: 4.3906e+02, l2_reg: 1.1951e+02\n",
      "2020/09/13, 10:46:28, Iteration: 64500, Train Loss: 4.2805e-01, c1: 5.1621e-02, c2: 5.0723e-02, d1: 1.6048e-02, d2: 6.3772e-03, l1_reg: 4.3932e+02, l2_reg: 1.1974e+02\n",
      "2020/09/13, 10:46:33, Iteration: 65000, Train Loss: 4.4487e-01, c1: 5.4163e-02, c2: 4.9845e-02, d1: 1.6009e-02, d2: 8.1075e-03, l1_reg: 4.3961e+02, l2_reg: 1.1997e+02\n",
      "2020/09/13, 10:46:37, Iteration: 65500, Train Loss: 4.3317e-01, c1: 5.2058e-02, c2: 4.9105e-02, d1: 1.6133e-02, d2: 7.2464e-03, l1_reg: 4.3989e+02, l2_reg: 1.2020e+02\n",
      "2020/09/13, 10:46:42, Iteration: 66000, Train Loss: 4.9501e-01, c1: 5.9592e-02, c2: 4.9912e-02, d1: 1.5678e-02, d2: 1.2890e-02, l1_reg: 4.4019e+02, l2_reg: 1.2044e+02\n",
      "2020/09/13, 10:46:47, Iteration: 66500, Train Loss: 4.1816e-01, c1: 5.5074e-02, c2: 4.7328e-02, d1: 1.5789e-02, d2: 6.3205e-03, l1_reg: 4.4044e+02, l2_reg: 1.2066e+02\n",
      "2020/09/13, 10:46:52, Iteration: 67000, Train Loss: 3.7435e-01, c1: 3.1592e-02, c2: 4.5915e-02, d1: 1.5495e-02, d2: 5.0052e-03, l1_reg: 4.4066e+02, l2_reg: 1.2087e+02\n",
      "2020/09/13, 10:46:57, Iteration: 67500, Train Loss: 4.0968e-01, c1: 4.9225e-02, c2: 4.8299e-02, d1: 1.5828e-02, d2: 5.7275e-03, l1_reg: 4.4099e+02, l2_reg: 1.2113e+02\n",
      "2020/09/13, 10:47:01, Iteration: 68000, Train Loss: 4.5138e-01, c1: 6.2393e-02, c2: 5.0026e-02, d1: 1.5762e-02, d2: 8.1277e-03, l1_reg: 4.4128e+02, l2_reg: 1.2136e+02\n",
      "2020/09/13, 10:47:06, Iteration: 68500, Train Loss: 5.0325e-01, c1: 4.9668e-02, c2: 4.7633e-02, d1: 1.5497e-02, d2: 1.5570e-02, l1_reg: 4.4139e+02, l2_reg: 1.2155e+02\n",
      "2020/09/13, 10:47:11, Iteration: 69000, Train Loss: 3.8804e-01, c1: 4.1081e-02, c2: 4.9679e-02, d1: 1.5827e-02, d2: 3.9646e-03, l1_reg: 4.4166e+02, l2_reg: 1.2178e+02\n",
      "2020/09/13, 10:47:15, Iteration: 69500, Train Loss: 4.0556e-01, c1: 5.0134e-02, c2: 4.9956e-02, d1: 1.5245e-02, d2: 5.3107e-03, l1_reg: 4.4198e+02, l2_reg: 1.2203e+02\n",
      "2020/09/13, 10:47:20, Iteration: 70000, Train Loss: 4.2904e-01, c1: 4.9959e-02, c2: 4.5023e-02, d1: 1.5552e-02, d2: 8.8487e-03, l1_reg: 4.4212e+02, l2_reg: 1.2222e+02\n",
      "2020/09/13, 10:47:25, Iteration: 70500, Train Loss: 4.7063e-01, c1: 6.5597e-02, c2: 5.3604e-02, d1: 1.5324e-02, d2: 9.0982e-03, l1_reg: 4.4227e+02, l2_reg: 1.2242e+02\n",
      "2020/09/13, 10:47:29, Iteration: 71000, Train Loss: 3.5104e-01, c1: 3.2670e-02, c2: 4.4405e-02, d1: 1.4986e-02, d2: 3.5293e-03, l1_reg: 4.4254e+02, l2_reg: 1.2267e+02\n",
      "2020/09/13, 10:47:34, Iteration: 71500, Train Loss: 3.9205e-01, c1: 3.7425e-02, c2: 4.4155e-02, d1: 1.5214e-02, d2: 7.0011e-03, l1_reg: 4.4277e+02, l2_reg: 1.2289e+02\n",
      "2020/09/13, 10:47:39, Iteration: 72000, Train Loss: 3.8435e-01, c1: 5.3115e-02, c2: 4.7704e-02, d1: 1.5268e-02, d2: 3.5431e-03, l1_reg: 4.4299e+02, l2_reg: 1.2312e+02\n",
      "2020/09/13, 10:47:43, Iteration: 72500, Train Loss: 3.5397e-01, c1: 3.5744e-02, c2: 4.4117e-02, d1: 1.5211e-02, d2: 3.3762e-03, l1_reg: 4.4316e+02, l2_reg: 1.2334e+02\n",
      "2020/09/13, 10:47:48, Iteration: 73000, Train Loss: 3.7360e-01, c1: 5.5015e-02, c2: 4.7952e-02, d1: 1.5427e-02, d2: 2.0454e-03, l1_reg: 4.4342e+02, l2_reg: 1.2358e+02\n",
      "2020/09/13, 10:47:53, Iteration: 73500, Train Loss: 4.4052e-01, c1: 5.4554e-02, c2: 4.8994e-02, d1: 1.5420e-02, d2: 8.4784e-03, l1_reg: 4.4368e+02, l2_reg: 1.2381e+02\n",
      "2020/09/13, 10:47:58, Iteration: 74000, Train Loss: 4.1751e-01, c1: 4.6646e-02, c2: 4.5974e-02, d1: 1.4890e-02, d2: 8.4040e-03, l1_reg: 4.4392e+02, l2_reg: 1.2403e+02\n",
      "2020/09/13, 10:48:02, Iteration: 74500, Train Loss: 3.6542e-01, c1: 4.8319e-02, c2: 4.6283e-02, d1: 1.4866e-02, d2: 2.9589e-03, l1_reg: 4.4420e+02, l2_reg: 1.2426e+02\n",
      "2020/09/13, 10:48:07, Iteration: 75000, Train Loss: 3.7341e-01, c1: 4.6967e-02, c2: 4.8875e-02, d1: 1.5432e-02, d2: 2.5488e-03, l1_reg: 4.4446e+02, l2_reg: 1.2449e+02\n",
      "2020/09/13, 10:48:12, Iteration: 75500, Train Loss: 4.5147e-01, c1: 4.2964e-02, c2: 4.5192e-02, d1: 1.5322e-02, d2: 1.1971e-02, l1_reg: 4.4470e+02, l2_reg: 1.2471e+02\n",
      "2020/09/13, 10:48:17, Iteration: 76000, Train Loss: 3.7190e-01, c1: 5.0889e-02, c2: 4.5397e-02, d1: 1.4943e-02, d2: 3.5384e-03, l1_reg: 4.4503e+02, l2_reg: 1.2497e+02\n",
      "2020/09/13, 10:48:21, Iteration: 76500, Train Loss: 4.3691e-01, c1: 3.3615e-02, c2: 4.6636e-02, d1: 1.5350e-02, d2: 1.0988e-02, l1_reg: 4.4523e+02, l2_reg: 1.2516e+02\n",
      "2020/09/13, 10:48:26, Iteration: 77000, Train Loss: 4.4866e-01, c1: 5.1345e-02, c2: 4.7046e-02, d1: 1.5092e-02, d2: 1.0525e-02, l1_reg: 4.4546e+02, l2_reg: 1.2539e+02\n",
      "2020/09/13, 10:48:31, Iteration: 77500, Train Loss: 3.7563e-01, c1: 4.5380e-02, c2: 4.7333e-02, d1: 1.4763e-02, d2: 4.0609e-03, l1_reg: 4.4577e+02, l2_reg: 1.2563e+02\n",
      "2020/09/13, 10:48:36, Iteration: 78000, Train Loss: 3.7953e-01, c1: 5.8182e-02, c2: 4.6812e-02, d1: 1.4900e-02, d2: 3.1899e-03, l1_reg: 4.4601e+02, l2_reg: 1.2585e+02\n",
      "2020/09/13, 10:48:40, Iteration: 78500, Train Loss: 3.8040e-01, c1: 4.6821e-02, c2: 4.7837e-02, d1: 1.4505e-02, d2: 4.5005e-03, l1_reg: 4.4621e+02, l2_reg: 1.2606e+02\n",
      "2020/09/13, 10:48:45, Iteration: 79000, Train Loss: 4.4525e-01, c1: 5.7752e-02, c2: 4.7912e-02, d1: 1.4729e-02, d2: 9.6464e-03, l1_reg: 4.4652e+02, l2_reg: 1.2632e+02\n",
      "2020/09/13, 10:48:50, Iteration: 79500, Train Loss: 3.8224e-01, c1: 6.3085e-02, c2: 4.4483e-02, d1: 1.4929e-02, d2: 3.6404e-03, l1_reg: 4.4680e+02, l2_reg: 1.2654e+02\n",
      "2020/09/13, 10:48:55, Iteration: 80000, Train Loss: 4.1044e-01, c1: 3.7690e-02, c2: 4.7167e-02, d1: 1.5075e-02, d2: 8.0498e-03, l1_reg: 4.4705e+02, l2_reg: 1.2676e+02\n",
      "2020/09/13, 10:48:59, Iteration: 80500, Train Loss: 3.4413e-01, c1: 4.0244e-02, c2: 4.3298e-02, d1: 1.4238e-02, d2: 3.1614e-03, l1_reg: 4.4733e+02, l2_reg: 1.2700e+02\n",
      "2020/09/13, 10:49:04, Iteration: 81000, Train Loss: 3.9492e-01, c1: 2.9807e-02, c2: 4.2531e-02, d1: 1.5148e-02, d2: 8.6038e-03, l1_reg: 4.4757e+02, l2_reg: 1.2722e+02\n",
      "2020/09/13, 10:49:09, Iteration: 81500, Train Loss: 3.4640e-01, c1: 4.6651e-02, c2: 4.7408e-02, d1: 1.4619e-02, d2: 1.1328e-03, l1_reg: 4.4784e+02, l2_reg: 1.2745e+02\n",
      "2020/09/13, 10:49:13, Iteration: 82000, Train Loss: 4.0021e-01, c1: 5.4624e-02, c2: 4.6391e-02, d1: 1.4455e-02, d2: 6.1858e-03, l1_reg: 4.4814e+02, l2_reg: 1.2770e+02\n",
      "2020/09/13, 10:49:18, Iteration: 82500, Train Loss: 4.0870e-01, c1: 3.1706e-02, c2: 4.5432e-02, d1: 1.4955e-02, d2: 9.1135e-03, l1_reg: 4.4837e+02, l2_reg: 1.2790e+02\n",
      "2020/09/13, 10:49:23, Iteration: 83000, Train Loss: 4.2241e-01, c1: 4.1086e-02, c2: 4.4506e-02, d1: 1.4596e-02, d2: 1.0184e-02, l1_reg: 4.4870e+02, l2_reg: 1.2814e+02\n",
      "2020/09/13, 10:49:28, Iteration: 83500, Train Loss: 4.0928e-01, c1: 4.9611e-02, c2: 4.3313e-02, d1: 1.4534e-02, d2: 8.4382e-03, l1_reg: 4.4894e+02, l2_reg: 1.2835e+02\n",
      "2020/09/13, 10:49:33, Iteration: 84000, Train Loss: 4.1699e-01, c1: 4.0953e-02, c2: 4.4858e-02, d1: 1.4562e-02, d2: 9.5837e-03, l1_reg: 4.4916e+02, l2_reg: 1.2855e+02\n",
      "2020/09/13, 10:49:37, Iteration: 84500, Train Loss: 3.5487e-01, c1: 4.0194e-02, c2: 4.3193e-02, d1: 1.4014e-02, d2: 4.4952e-03, l1_reg: 4.4941e+02, l2_reg: 1.2876e+02\n",
      "2020/09/13, 10:49:42, Iteration: 85000, Train Loss: 3.5166e-01, c1: 4.5535e-02, c2: 4.5754e-02, d1: 1.4055e-02, d2: 2.8308e-03, l1_reg: 4.4969e+02, l2_reg: 1.2899e+02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/13, 10:49:47, Iteration: 85500, Train Loss: 3.6990e-01, c1: 4.5551e-02, c2: 4.5205e-02, d1: 1.5385e-02, d2: 3.4874e-03, l1_reg: 4.4990e+02, l2_reg: 1.2920e+02\n",
      "2020/09/13, 10:49:51, Iteration: 86000, Train Loss: 3.6020e-01, c1: 3.6172e-02, c2: 4.5396e-02, d1: 1.3913e-02, d2: 4.8707e-03, l1_reg: 4.5019e+02, l2_reg: 1.2943e+02\n",
      "2020/09/13, 10:49:56, Iteration: 86500, Train Loss: 3.8485e-01, c1: 4.8459e-02, c2: 4.2187e-02, d1: 1.3936e-02, d2: 7.0470e-03, l1_reg: 4.5044e+02, l2_reg: 1.2966e+02\n",
      "2020/09/13, 10:50:01, Iteration: 87000, Train Loss: 3.8470e-01, c1: 5.1100e-02, c2: 4.4350e-02, d1: 1.3828e-02, d2: 6.2264e-03, l1_reg: 4.5068e+02, l2_reg: 1.2986e+02\n",
      "2020/09/13, 10:50:05, Iteration: 87500, Train Loss: 3.9252e-01, c1: 5.1178e-02, c2: 4.7202e-02, d1: 1.4574e-02, d2: 5.3985e-03, l1_reg: 4.5100e+02, l2_reg: 1.3011e+02\n",
      "2020/09/13, 10:50:10, Iteration: 88000, Train Loss: 4.1758e-01, c1: 4.5205e-02, c2: 4.6885e-02, d1: 1.3721e-02, d2: 9.4506e-03, l1_reg: 4.5123e+02, l2_reg: 1.3031e+02\n",
      "2020/09/13, 10:50:15, Iteration: 88500, Train Loss: 3.4756e-01, c1: 3.7742e-02, c2: 4.5312e-02, d1: 1.4386e-02, d2: 3.0013e-03, l1_reg: 4.5151e+02, l2_reg: 1.3057e+02\n",
      "2020/09/13, 10:50:20, Iteration: 89000, Train Loss: 3.8649e-01, c1: 4.0739e-02, c2: 4.5101e-02, d1: 1.4668e-02, d2: 6.3760e-03, l1_reg: 4.5166e+02, l2_reg: 1.3073e+02\n",
      "2020/09/13, 10:50:25, Iteration: 89500, Train Loss: 3.5777e-01, c1: 5.6098e-02, c2: 4.3846e-02, d1: 1.4120e-02, d2: 2.8923e-03, l1_reg: 4.5190e+02, l2_reg: 1.3095e+02\n",
      "2020/09/13, 10:50:29, Iteration: 90000, Train Loss: 3.4570e-01, c1: 4.5896e-02, c2: 4.4583e-02, d1: 1.3878e-02, d2: 2.7270e-03, l1_reg: 4.5211e+02, l2_reg: 1.3116e+02\n",
      "2020/09/13, 10:50:34, Iteration: 90500, Train Loss: 3.5769e-01, c1: 4.5223e-02, c2: 4.5324e-02, d1: 1.3978e-02, d2: 3.6711e-03, l1_reg: 4.5228e+02, l2_reg: 1.3135e+02\n",
      "2020/09/13, 10:50:39, Iteration: 91000, Train Loss: 4.4784e-01, c1: 6.4986e-02, c2: 4.6693e-02, d1: 1.4945e-02, d2: 9.3318e-03, l1_reg: 4.5248e+02, l2_reg: 1.3156e+02\n",
      "2020/09/13, 10:50:43, Iteration: 91500, Train Loss: 3.7829e-01, c1: 5.6293e-02, c2: 4.4649e-02, d1: 1.3452e-02, d2: 5.3519e-03, l1_reg: 4.5277e+02, l2_reg: 1.3180e+02\n",
      "2020/09/13, 10:50:48, Iteration: 92000, Train Loss: 3.8396e-01, c1: 5.4135e-02, c2: 4.3451e-02, d1: 1.4570e-02, d2: 5.3771e-03, l1_reg: 4.5295e+02, l2_reg: 1.3200e+02\n",
      "2020/09/13, 10:50:53, Iteration: 92500, Train Loss: 4.0620e-01, c1: 5.3143e-02, c2: 4.6221e-02, d1: 1.4546e-02, d2: 6.8926e-03, l1_reg: 4.5320e+02, l2_reg: 1.3223e+02\n",
      "2020/09/13, 10:50:57, Iteration: 93000, Train Loss: 3.6708e-01, c1: 5.1260e-02, c2: 4.3982e-02, d1: 1.3671e-02, d2: 4.7157e-03, l1_reg: 4.5335e+02, l2_reg: 1.3241e+02\n",
      "2020/09/13, 10:51:02, Iteration: 93500, Train Loss: 3.7489e-01, c1: 4.2143e-02, c2: 4.4300e-02, d1: 1.3821e-02, d2: 6.1624e-03, l1_reg: 4.5358e+02, l2_reg: 1.3262e+02\n",
      "2020/09/13, 10:51:07, Iteration: 94000, Train Loss: 3.2659e-01, c1: 4.7112e-02, c2: 4.2016e-02, d1: 1.3864e-02, d2: 1.4783e-03, l1_reg: 4.5385e+02, l2_reg: 1.3285e+02\n",
      "2020/09/13, 10:51:11, Iteration: 94500, Train Loss: 3.6573e-01, c1: 5.8279e-02, c2: 4.8641e-02, d1: 1.3947e-02, d2: 2.2052e-03, l1_reg: 4.5415e+02, l2_reg: 1.3309e+02\n",
      "2020/09/13, 10:51:16, Iteration: 95000, Train Loss: 4.0586e-01, c1: 4.3564e-02, c2: 4.3544e-02, d1: 1.3836e-02, d2: 9.3297e-03, l1_reg: 4.5439e+02, l2_reg: 1.3332e+02\n",
      "2020/09/13, 10:51:21, Iteration: 95500, Train Loss: 3.7193e-01, c1: 4.9150e-02, c2: 4.5155e-02, d1: 1.3569e-02, d2: 5.1614e-03, l1_reg: 4.5454e+02, l2_reg: 1.3351e+02\n",
      "2020/09/13, 10:51:26, Iteration: 96000, Train Loss: 3.5184e-01, c1: 3.3121e-02, c2: 4.1184e-02, d1: 1.3247e-02, d2: 6.2691e-03, l1_reg: 4.5475e+02, l2_reg: 1.3371e+02\n",
      "2020/09/13, 10:51:30, Iteration: 96500, Train Loss: 4.3562e-01, c1: 5.3634e-02, c2: 4.5703e-02, d1: 1.4115e-02, d2: 1.0373e-02, l1_reg: 4.5495e+02, l2_reg: 1.3391e+02\n",
      "2020/09/13, 10:51:35, Iteration: 97000, Train Loss: 3.9982e-01, c1: 4.7452e-02, c2: 4.3939e-02, d1: 1.3936e-02, d2: 8.1178e-03, l1_reg: 4.5515e+02, l2_reg: 1.3411e+02\n",
      "2020/09/13, 10:51:40, Iteration: 97500, Train Loss: 3.8439e-01, c1: 3.6615e-02, c2: 4.6243e-02, d1: 1.3483e-02, d2: 7.4216e-03, l1_reg: 4.5550e+02, l2_reg: 1.3437e+02\n",
      "2020/09/13, 10:51:44, Iteration: 98000, Train Loss: 3.6882e-01, c1: 4.7686e-02, c2: 4.1797e-02, d1: 1.2998e-02, d2: 6.5755e-03, l1_reg: 4.5567e+02, l2_reg: 1.3457e+02\n",
      "2020/09/13, 10:51:49, Iteration: 98500, Train Loss: 3.4136e-01, c1: 5.1017e-02, c2: 4.2362e-02, d1: 1.3068e-02, d2: 3.2564e-03, l1_reg: 4.5588e+02, l2_reg: 1.3478e+02\n",
      "2020/09/13, 10:51:54, Iteration: 99000, Train Loss: 3.5803e-01, c1: 4.7931e-02, c2: 4.4995e-02, d1: 1.3041e-02, d2: 4.4707e-03, l1_reg: 4.5615e+02, l2_reg: 1.3502e+02\n",
      "2020/09/13, 10:51:58, Iteration: 99500, Train Loss: 3.4185e-01, c1: 5.1649e-02, c2: 4.4492e-02, d1: 1.3667e-02, d2: 2.0051e-03, l1_reg: 4.5633e+02, l2_reg: 1.3521e+02\n",
      "2020/09/13, 10:52:03, Iteration: 100000, Train Loss: 4.0211e-01, c1: 5.3247e-02, c2: 4.7879e-02, d1: 1.3202e-02, d2: 7.3204e-03, l1_reg: 4.5648e+02, l2_reg: 1.3539e+02\n"
     ]
    }
   ],
   "source": [
    "import jax\n",
    "import jax.numpy as jnp\n",
    "from jax import random\n",
    "\n",
    "import os, sys\n",
    "sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(\"./\"))))\n",
    "\n",
    "from jaxmeta.model_init import init_siren_params\n",
    "\n",
    "from training import Agent\n",
    "from loss import model, loss_fn, evaluate_fn\n",
    "from data import generate_dataset, generate_batch_fn\n",
    "import config\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "name = \"1\"\n",
    "\n",
    "config.PATH = \"experiments_2\"\n",
    "config.NAME = \"{}/{}\".format(config.PATH, name)\n",
    "config.log_file = sys.stdout\n",
    "\n",
    "config.layers = [2] + [32]*4 + [2]\n",
    "config.c0 = 1.0\n",
    "config.w0 = jnp.array([[1.0, 1.0]]).T\n",
    "\n",
    "key, *subkeys = random.split(config.key, 3)\n",
    "params = init_siren_params(subkeys[0], config.layers, config.c0, config.w0)\n",
    "\n",
    "config.n_data = {\n",
    "\t\"i\": 160, \"b\": 160, \"cx\": 160, \"ct\": 160,\n",
    "}\n",
    "config.batch_size = {\n",
    "\t\"dirichlet\": (config.n_data[\"i\"] + 2*config.n_data[\"b\"]),\n",
    "\t\"collocation\": (config.n_data[\"cx\"]*config.n_data[\"ct\"]),\n",
    "}\n",
    "config.weights = {\n",
    "\t\"c1\": 1.0,\n",
    "\t\"c2\": 3.0,\n",
    "\t\"d1\": 10.0,\n",
    "\t\"d2\": 10.0,\n",
    "\t\"l1\": 1e-8,\n",
    "\t\"l2\": 1e-8,\n",
    "}\n",
    "\n",
    "datasets = generate_dataset(config.n_data[\"i\"], config.n_data[\"b\"], config.n_data[\"cx\"], config.n_data[\"ct\"])\n",
    "batch_fn, evaluate_batch_fn = generate_batch_fn(subkeys[1], config.batch_size, *datasets, config.weights)\n",
    "\n",
    "config.iterations = 100000\n",
    "config.print_every = 500\n",
    "config.lr = 1e-3\n",
    "\n",
    "agent = Agent(params, loss_fn, evaluate_fn, \"models/{}\".format(config.NAME))\n",
    "agent.compile(config.optimizer, config.lr)\n",
    "agent.train(config.iterations, batch_fn, evaluate_batch_fn, config.print_every, config.save_every, config.loss_names, config.log_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 370,\n",
      " njev: 370,\n",
      " nit: 326\n",
      "2020/09/13, 11:12:19, Batch: 0, BFGS Loss: 2.3907e-01, c1: 1.2939e-02, c2: 3.3986e-02, d1: 1.2228e-02, d2: 1.8805e-04, l1_reg: 4.5858e+02, l2_reg: 1.3735e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 367,\n",
      " njev: 367,\n",
      " nit: 289\n",
      "2020/09/13, 11:12:32, Batch: 1, BFGS Loss: 2.2856e-01, c1: 1.1241e-02, c2: 3.3126e-02, d1: 1.1619e-02, d2: 1.7436e-04, l1_reg: 4.6048e+02, l2_reg: 1.3900e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 249,\n",
      " njev: 249,\n",
      " nit: 179\n",
      "2020/09/13, 11:12:43, Batch: 2, BFGS Loss: 2.2437e-01, c1: 9.8938e-03, c2: 3.2565e-02, d1: 1.1545e-02, d2: 1.3283e-04, l1_reg: 4.6101e+02, l2_reg: 1.3947e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 232,\n",
      " njev: 232,\n",
      " nit: 147\n",
      "2020/09/13, 11:12:53, Batch: 3, BFGS Loss: 2.2239e-01, c1: 9.9350e-03, c2: 3.2401e-02, d1: 1.1394e-02, d2: 1.3100e-04, l1_reg: 4.6160e+02, l2_reg: 1.3994e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 223,\n",
      " njev: 223,\n",
      " nit: 138\n",
      "2020/09/13, 11:13:04, Batch: 4, BFGS Loss: 2.2093e-01, c1: 9.5315e-03, c2: 3.2208e-02, d1: 1.1358e-02, d2: 1.1886e-04, l1_reg: 4.6186e+02, l2_reg: 1.4016e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 86,\n",
      " njev: 86,\n",
      " nit: 16\n",
      "2020/09/13, 11:13:12, Batch: 5, BFGS Loss: 2.2101e-01, c1: 9.5813e-03, c2: 3.2211e-02, d1: 1.1359e-02, d2: 1.2039e-04, l1_reg: 4.6186e+02, l2_reg: 1.4016e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 252,\n",
      " njev: 252,\n",
      " nit: 155\n",
      "2020/09/13, 11:13:23, Batch: 6, BFGS Loss: 2.1881e-01, c1: 1.0233e-02, c2: 3.1901e-02, d1: 1.1141e-02, d2: 1.4580e-04, l1_reg: 4.6294e+02, l2_reg: 1.4102e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 251,\n",
      " njev: 251,\n",
      " nit: 174\n",
      "2020/09/13, 11:13:35, Batch: 7, BFGS Loss: 2.1596e-01, c1: 9.3498e-03, c2: 3.1597e-02, d1: 1.1071e-02, d2: 1.1025e-04, l1_reg: 4.6338e+02, l2_reg: 1.4140e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 260,\n",
      " njev: 260,\n",
      " nit: 172\n",
      "2020/09/13, 11:13:46, Batch: 8, BFGS Loss: 2.1374e-01, c1: 9.5946e-03, c2: 3.1446e-02, d1: 1.0876e-02, d2: 1.0418e-04, l1_reg: 4.6421e+02, l2_reg: 1.4208e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 258,\n",
      " njev: 258,\n",
      " nit: 176\n",
      "2020/09/13, 11:13:57, Batch: 9, BFGS Loss: 2.1150e-01, c1: 9.2479e-03, c2: 3.1079e-02, d1: 1.0811e-02, d2: 8.9434e-05, l1_reg: 4.6478e+02, l2_reg: 1.4257e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 254,\n",
      " njev: 254,\n",
      " nit: 159\n",
      "2020/09/13, 11:14:07, Batch: 10, BFGS Loss: 2.0986e-01, c1: 9.0575e-03, c2: 3.0986e-02, d1: 1.0705e-02, d2: 7.8771e-05, l1_reg: 4.6519e+02, l2_reg: 1.4290e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 233,\n",
      " njev: 233,\n",
      " nit: 122\n",
      "2020/09/13, 11:14:18, Batch: 11, BFGS Loss: 2.0934e-01, c1: 8.9908e-03, c2: 3.0826e-02, d1: 1.0679e-02, d2: 1.0723e-04, l1_reg: 4.6543e+02, l2_reg: 1.4311e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 211,\n",
      " njev: 211,\n",
      " nit: 115\n",
      "2020/09/13, 11:14:28, Batch: 12, BFGS Loss: 2.0823e-01, c1: 8.8439e-03, c2: 3.0808e-02, d1: 1.0622e-02, d2: 7.3789e-05, l1_reg: 4.6564e+02, l2_reg: 1.4329e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 64,\n",
      " njev: 64,\n",
      " nit: 8\n",
      "2020/09/13, 11:14:37, Batch: 13, BFGS Loss: 2.0949e-01, c1: 9.1792e-03, c2: 3.0831e-02, d1: 1.0628e-02, d2: 1.5331e-04, l1_reg: 4.6564e+02, l2_reg: 1.4329e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 222,\n",
      " njev: 222,\n",
      " nit: 142\n",
      "2020/09/13, 11:14:48, Batch: 14, BFGS Loss: 2.0705e-01, c1: 9.0725e-03, c2: 3.0607e-02, d1: 1.0546e-02, d2: 6.8494e-05, l1_reg: 4.6615e+02, l2_reg: 1.4372e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 232,\n",
      " njev: 232,\n",
      " nit: 139\n",
      "2020/09/13, 11:14:58, Batch: 15, BFGS Loss: 2.0586e-01, c1: 8.7111e-03, c2: 3.0577e-02, d1: 1.0479e-02, d2: 6.2417e-05, l1_reg: 4.6640e+02, l2_reg: 1.4393e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 39,\n",
      " njev: 39,\n",
      " nit: 4\n",
      "2020/09/13, 11:15:07, Batch: 16, BFGS Loss: 2.0862e-01, c1: 9.6158e-03, c2: 3.0630e-02, d1: 1.0533e-02, d2: 1.7838e-04, l1_reg: 4.6640e+02, l2_reg: 1.4393e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 221,\n",
      " njev: 221,\n",
      " nit: 131\n",
      "2020/09/13, 11:15:17, Batch: 17, BFGS Loss: 2.0507e-01, c1: 8.8136e-03, c2: 3.0408e-02, d1: 1.0439e-02, d2: 6.2952e-05, l1_reg: 4.6673e+02, l2_reg: 1.4421e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 40,\n",
      " njev: 40,\n",
      " nit: 5\n",
      "2020/09/13, 11:15:26, Batch: 18, BFGS Loss: 2.1058e-01, c1: 1.2101e-02, c2: 3.0551e-02, d1: 1.0522e-02, d2: 1.5988e-04, l1_reg: 4.6673e+02, l2_reg: 1.4421e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 222,\n",
      " njev: 222,\n",
      " nit: 144\n",
      "2020/09/13, 11:15:37, Batch: 19, BFGS Loss: 2.0394e-01, c1: 8.7512e-03, c2: 3.0317e-02, d1: 1.0360e-02, d2: 6.2723e-05, l1_reg: 4.6712e+02, l2_reg: 1.4454e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 252,\n",
      " njev: 252,\n",
      " nit: 141\n",
      "2020/09/13, 11:15:48, Batch: 20, BFGS Loss: 2.0298e-01, c1: 8.6516e-03, c2: 3.0147e-02, d1: 1.0330e-02, d2: 5.7957e-05, l1_reg: 4.6738e+02, l2_reg: 1.4478e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 45,\n",
      " njev: 45,\n",
      " nit: 6\n",
      "2020/09/13, 11:15:56, Batch: 21, BFGS Loss: 2.0458e-01, c1: 9.6719e-03, c2: 3.0192e-02, d1: 1.0334e-02, d2: 9.9206e-05, l1_reg: 4.6738e+02, l2_reg: 1.4478e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 101,\n",
      " njev: 101,\n",
      " nit: 28\n",
      "2020/09/13, 11:16:05, Batch: 22, BFGS Loss: 2.0349e-01, c1: 8.9511e-03, c2: 3.0168e-02, d1: 1.0331e-02, d2: 7.2500e-05, l1_reg: 4.6738e+02, l2_reg: 1.4478e+02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 191,\n",
      " njev: 191,\n",
      " nit: 81\n",
      "2020/09/13, 11:16:15, Batch: 23, BFGS Loss: 2.0256e-01, c1: 8.5041e-03, c2: 3.0111e-02, d1: 1.0313e-02, d2: 5.7970e-05, l1_reg: 4.6747e+02, l2_reg: 1.4485e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 212,\n",
      " njev: 212,\n",
      " nit: 100\n",
      "2020/09/13, 11:16:25, Batch: 24, BFGS Loss: 2.0211e-01, c1: 8.5313e-03, c2: 3.0043e-02, d1: 1.0282e-02, d2: 6.2971e-05, l1_reg: 4.6767e+02, l2_reg: 1.4503e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 198,\n",
      " njev: 198,\n",
      " nit: 96\n",
      "2020/09/13, 11:16:36, Batch: 25, BFGS Loss: 2.0169e-01, c1: 8.4535e-03, c2: 2.9992e-02, d1: 1.0270e-02, d2: 5.4685e-05, l1_reg: 4.6775e+02, l2_reg: 1.4510e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 31,\n",
      " njev: 31,\n",
      " nit: 3\n",
      "2020/09/13, 11:16:44, Batch: 26, BFGS Loss: 2.0588e-01, c1: 1.0380e-02, c2: 3.0057e-02, d1: 1.0315e-02, d2: 2.1764e-04, l1_reg: 4.6775e+02, l2_reg: 1.4510e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 170,\n",
      " njev: 170,\n",
      " nit: 86\n",
      "2020/09/13, 11:16:54, Batch: 27, BFGS Loss: 2.0149e-01, c1: 8.4322e-03, c2: 2.9979e-02, d1: 1.0256e-02, d2: 5.5429e-05, l1_reg: 4.6782e+02, l2_reg: 1.4516e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 33,\n",
      " njev: 33,\n",
      " nit: 3\n",
      "2020/09/13, 11:17:03, Batch: 28, BFGS Loss: 2.0447e-01, c1: 9.8050e-03, c2: 3.0029e-02, d1: 1.0338e-02, d2: 1.1887e-04, l1_reg: 4.6782e+02, l2_reg: 1.4516e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 212,\n",
      " njev: 212,\n",
      " nit: 127\n",
      "2020/09/13, 11:17:14, Batch: 29, BFGS Loss: 1.9983e-01, c1: 8.8968e-03, c2: 2.9624e-02, d1: 1.0128e-02, d2: 7.7470e-05, l1_reg: 4.6884e+02, l2_reg: 1.4606e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 214,\n",
      " njev: 214,\n",
      " nit: 123\n",
      "2020/09/13, 11:17:24, Batch: 30, BFGS Loss: 1.9826e-01, c1: 8.2193e-03, c2: 2.9517e-02, d1: 1.0091e-02, d2: 5.6766e-05, l1_reg: 4.6894e+02, l2_reg: 1.4614e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 66,\n",
      " njev: 66,\n",
      " nit: 9\n",
      "2020/09/13, 11:17:33, Batch: 31, BFGS Loss: 2.0070e-01, c1: 9.6254e-03, c2: 2.9672e-02, d1: 1.0131e-02, d2: 7.4081e-05, l1_reg: 4.6894e+02, l2_reg: 1.4614e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 114,\n",
      " njev: 114,\n",
      " nit: 32\n",
      "2020/09/13, 11:17:42, Batch: 32, BFGS Loss: 1.9823e-01, c1: 8.2121e-03, c2: 2.9515e-02, d1: 1.0092e-02, d2: 5.4982e-05, l1_reg: 4.6894e+02, l2_reg: 1.4614e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 17,\n",
      " njev: 17,\n",
      " nit: 1\n",
      "2020/09/13, 11:17:50, Batch: 33, BFGS Loss: 2.9753e-01, c1: 1.5286e-02, c2: 3.0783e-02, d1: 1.0183e-02, d2: 8.8053e-03, l1_reg: 4.6894e+02, l2_reg: 1.4615e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 181,\n",
      " njev: 181,\n",
      " nit: 92\n",
      "2020/09/13, 11:18:01, Batch: 34, BFGS Loss: 1.9803e-01, c1: 8.2639e-03, c2: 2.9486e-02, d1: 1.0075e-02, d2: 5.5313e-05, l1_reg: 4.6904e+02, l2_reg: 1.4623e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 40,\n",
      " njev: 40,\n",
      " nit: 5\n",
      "2020/09/13, 11:18:09, Batch: 35, BFGS Loss: 1.9936e-01, c1: 9.0601e-03, c2: 2.9547e-02, d1: 1.0083e-02, d2: 8.2105e-05, l1_reg: 4.6904e+02, l2_reg: 1.4623e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 198,\n",
      " njev: 198,\n",
      " nit: 106\n",
      "2020/09/13, 11:18:19, Batch: 36, BFGS Loss: 1.9767e-01, c1: 8.2144e-03, c2: 2.9477e-02, d1: 1.0044e-02, d2: 5.8166e-05, l1_reg: 4.6918e+02, l2_reg: 1.4635e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 43,\n",
      " njev: 43,\n",
      " nit: 5\n",
      "2020/09/13, 11:18:28, Batch: 37, BFGS Loss: 1.9904e-01, c1: 8.9182e-03, c2: 2.9529e-02, d1: 1.0048e-02, d2: 1.0467e-04, l1_reg: 4.6918e+02, l2_reg: 1.4635e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 191,\n",
      " njev: 191,\n",
      " nit: 94\n",
      "2020/09/13, 11:18:39, Batch: 38, BFGS Loss: 1.9741e-01, c1: 8.1799e-03, c2: 2.9396e-02, d1: 1.0049e-02, d2: 5.5186e-05, l1_reg: 4.6924e+02, l2_reg: 1.4640e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 37,\n",
      " njev: 37,\n",
      " nit: 4\n",
      "2020/09/13, 11:18:47, Batch: 39, BFGS Loss: 1.9872e-01, c1: 8.7753e-03, c2: 2.9455e-02, d1: 1.0057e-02, d2: 1.0060e-04, l1_reg: 4.6924e+02, l2_reg: 1.4640e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 107,\n",
      " njev: 107,\n",
      " nit: 26\n",
      "2020/09/13, 11:18:56, Batch: 40, BFGS Loss: 1.9744e-01, c1: 8.2031e-03, c2: 2.9396e-02, d1: 1.0049e-02, d2: 5.5773e-05, l1_reg: 4.6924e+02, l2_reg: 1.4640e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 140,\n",
      " njev: 140,\n",
      " nit: 27\n",
      "2020/09/13, 11:19:05, Batch: 41, BFGS Loss: 1.9786e-01, c1: 8.4549e-03, c2: 2.9416e-02, d1: 1.0049e-02, d2: 6.6273e-05, l1_reg: 4.6924e+02, l2_reg: 1.4640e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 111,\n",
      " njev: 111,\n",
      " nit: 28\n",
      "2020/09/13, 11:19:14, Batch: 42, BFGS Loss: 1.9742e-01, c1: 8.1990e-03, c2: 2.9392e-02, d1: 1.0049e-02, d2: 5.5087e-05, l1_reg: 4.6924e+02, l2_reg: 1.4640e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 131,\n",
      " njev: 131,\n",
      " nit: 26\n",
      "2020/09/13, 11:19:24, Batch: 43, BFGS Loss: 1.9738e-01, c1: 8.1693e-03, c2: 2.9393e-02, d1: 1.0049e-02, d2: 5.4216e-05, l1_reg: 4.6924e+02, l2_reg: 1.4640e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 76,\n",
      " njev: 76,\n",
      " nit: 10\n",
      "2020/09/13, 11:19:33, Batch: 44, BFGS Loss: 1.9809e-01, c1: 8.4071e-03, c2: 2.9415e-02, d1: 1.0053e-02, d2: 9.0942e-05, l1_reg: 4.6924e+02, l2_reg: 1.4640e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 115,\n",
      " njev: 115,\n",
      " nit: 29\n",
      "2020/09/13, 11:19:42, Batch: 45, BFGS Loss: 1.9738e-01, c1: 8.1678e-03, c2: 2.9393e-02, d1: 1.0049e-02, d2: 5.3957e-05, l1_reg: 4.6924e+02, l2_reg: 1.4641e+02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 18,\n",
      " njev: 18,\n",
      " nit: 1\n",
      "2020/09/13, 11:19:51, Batch: 46, BFGS Loss: 3.2028e-01, c1: 1.9619e-02, c2: 3.0221e-02, d1: 1.0610e-02, d2: 1.0389e-02, l1_reg: 4.6924e+02, l2_reg: 1.4640e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 114,\n",
      " njev: 114,\n",
      " nit: 36\n",
      "2020/09/13, 11:20:00, Batch: 47, BFGS Loss: 1.9746e-01, c1: 8.2209e-03, c2: 2.9394e-02, d1: 1.0048e-02, d2: 5.7038e-05, l1_reg: 4.6924e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 98,\n",
      " njev: 98,\n",
      " nit: 21\n",
      "2020/09/13, 11:20:09, Batch: 48, BFGS Loss: 1.9744e-01, c1: 8.2206e-03, c2: 2.9393e-02, d1: 1.0048e-02, d2: 5.5283e-05, l1_reg: 4.6924e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 117,\n",
      " njev: 117,\n",
      " nit: 22\n",
      "2020/09/13, 11:20:18, Batch: 49, BFGS Loss: 1.9740e-01, c1: 8.1779e-03, c2: 2.9394e-02, d1: 1.0048e-02, d2: 5.5489e-05, l1_reg: 4.6924e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 110,\n",
      " njev: 110,\n",
      " nit: 18\n",
      "2020/09/13, 11:20:27, Batch: 50, BFGS Loss: 1.9763e-01, c1: 8.3075e-03, c2: 2.9407e-02, d1: 1.0049e-02, d2: 6.0780e-05, l1_reg: 4.6924e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 111,\n",
      " njev: 111,\n",
      " nit: 24\n",
      "2020/09/13, 11:20:36, Batch: 51, BFGS Loss: 1.9739e-01, c1: 8.1760e-03, c2: 2.9395e-02, d1: 1.0048e-02, d2: 5.4388e-05, l1_reg: 4.6924e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 95,\n",
      " njev: 95,\n",
      " nit: 15\n",
      "2020/09/13, 11:20:45, Batch: 52, BFGS Loss: 1.9748e-01, c1: 8.2408e-03, c2: 2.9399e-02, d1: 1.0048e-02, d2: 5.5015e-05, l1_reg: 4.6924e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 110,\n",
      " njev: 110,\n",
      " nit: 20\n",
      "2020/09/13, 11:20:55, Batch: 53, BFGS Loss: 1.9748e-01, c1: 8.2387e-03, c2: 2.9395e-02, d1: 1.0049e-02, d2: 5.6609e-05, l1_reg: 4.6924e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 123,\n",
      " njev: 123,\n",
      " nit: 25\n",
      "2020/09/13, 11:21:04, Batch: 54, BFGS Loss: 1.9739e-01, c1: 8.1722e-03, c2: 2.9393e-02, d1: 1.0048e-02, d2: 5.4918e-05, l1_reg: 4.6924e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 20,\n",
      " njev: 20,\n",
      " nit: 1\n",
      "2020/09/13, 11:21:12, Batch: 55, BFGS Loss: 3.5570e-01, c1: 3.2378e-02, c2: 3.3420e-02, d1: 1.0884e-02, d2: 1.1422e-02, l1_reg: 4.6925e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 118,\n",
      " njev: 118,\n",
      " nit: 43\n",
      "2020/09/13, 11:21:22, Batch: 56, BFGS Loss: 1.9736e-01, c1: 8.1615e-03, c2: 2.9392e-02, d1: 1.0048e-02, d2: 5.4095e-05, l1_reg: 4.6924e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 75,\n",
      " njev: 75,\n",
      " nit: 9\n",
      "2020/09/13, 11:21:31, Batch: 57, BFGS Loss: 1.9970e-01, c1: 9.9692e-03, c2: 2.9415e-02, d1: 1.0059e-02, d2: 8.8328e-05, l1_reg: 4.6924e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 108,\n",
      " njev: 108,\n",
      " nit: 34\n",
      "2020/09/13, 11:21:40, Batch: 58, BFGS Loss: 1.9736e-01, c1: 8.1599e-03, c2: 2.9392e-02, d1: 1.0048e-02, d2: 5.4068e-05, l1_reg: 4.6925e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 30,\n",
      " njev: 30,\n",
      " nit: 2\n",
      "2020/09/13, 11:21:49, Batch: 59, BFGS Loss: 2.2909e-01, c1: 1.4096e-02, c2: 3.0183e-02, d1: 1.0081e-02, d2: 2.3624e-03, l1_reg: 4.6925e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 114,\n",
      " njev: 114,\n",
      " nit: 39\n",
      "2020/09/13, 11:21:59, Batch: 60, BFGS Loss: 1.9736e-01, c1: 8.1595e-03, c2: 2.9392e-02, d1: 1.0047e-02, d2: 5.4187e-05, l1_reg: 4.6925e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 19,\n",
      " njev: 19,\n",
      " nit: 1\n",
      "2020/09/13, 11:22:07, Batch: 61, BFGS Loss: 2.8632e-01, c1: 1.3993e-02, c2: 3.0278e-02, d1: 1.0334e-02, d2: 7.8139e-03, l1_reg: 4.6925e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 119,\n",
      " njev: 119,\n",
      " nit: 41\n",
      "2020/09/13, 11:22:17, Batch: 62, BFGS Loss: 1.9735e-01, c1: 8.1574e-03, c2: 2.9392e-02, d1: 1.0047e-02, d2: 5.4116e-05, l1_reg: 4.6925e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 42,\n",
      " njev: 42,\n",
      " nit: 5\n",
      "2020/09/13, 11:22:26, Batch: 63, BFGS Loss: 2.0273e-01, c1: 1.0143e-02, c2: 2.9513e-02, d1: 1.0151e-02, d2: 2.5338e-04, l1_reg: 4.6925e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 122,\n",
      " njev: 122,\n",
      " nit: 37\n",
      "2020/09/13, 11:22:35, Batch: 64, BFGS Loss: 1.9746e-01, c1: 8.2612e-03, c2: 2.9394e-02, d1: 1.0046e-02, d2: 5.5149e-05, l1_reg: 4.6925e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 142,\n",
      " njev: 142,\n",
      " nit: 36\n",
      "2020/09/13, 11:22:45, Batch: 65, BFGS Loss: 1.9737e-01, c1: 8.1737e-03, c2: 2.9393e-02, d1: 1.0046e-02, d2: 5.5571e-05, l1_reg: 4.6925e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 115,\n",
      " njev: 115,\n",
      " nit: 20\n",
      "2020/09/13, 11:22:54, Batch: 66, BFGS Loss: 1.9882e-01, c1: 8.7304e-03, c2: 2.9441e-02, d1: 1.0048e-02, d2: 1.2827e-04, l1_reg: 4.6925e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 104,\n",
      " njev: 104,\n",
      " nit: 31\n",
      "2020/09/13, 11:23:03, Batch: 67, BFGS Loss: 1.9734e-01, c1: 8.1510e-03, c2: 2.9393e-02, d1: 1.0046e-02, d2: 5.4157e-05, l1_reg: 4.6925e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 58,\n",
      " njev: 58,\n",
      " nit: 7\n",
      "2020/09/13, 11:23:12, Batch: 68, BFGS Loss: 2.0120e-01, c1: 9.8764e-03, c2: 2.9611e-02, d1: 1.0077e-02, d2: 1.7185e-04, l1_reg: 4.6925e+02, l2_reg: 1.4641e+02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 110,\n",
      " njev: 110,\n",
      " nit: 33\n",
      "2020/09/13, 11:23:21, Batch: 69, BFGS Loss: 1.9733e-01, c1: 8.1516e-03, c2: 2.9392e-02, d1: 1.0046e-02, d2: 5.4088e-05, l1_reg: 4.6925e+02, l2_reg: 1.4641e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 18,\n",
      " njev: 18,\n",
      " nit: 1\n",
      "2020/09/13, 11:23:30, Batch: 70, BFGS Loss: 2.9326e-01, c1: 8.9109e-03, c2: 3.0212e-02, d1: 1.0085e-02, d2: 9.2855e-03, l1_reg: 4.6926e+02, l2_reg: 1.4642e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 20,\n",
      " njev: 20,\n",
      " nit: 2\n",
      "2020/09/13, 11:23:39, Batch: 71, BFGS Loss: 5.4106e+00, c1: 5.9691e-01, c2: 1.0671e-01, d1: 3.3910e-02, d2: 4.1544e-01, l1_reg: 4.6904e+02, l2_reg: 1.4619e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 26,\n",
      " njev: 26,\n",
      " nit: 5\n",
      "2020/09/13, 11:23:47, Batch: 72, BFGS Loss: 3.1403e+00, c1: 8.9370e-01, c2: 2.0764e-01, d1: 3.5053e-02, d2: 1.2732e-01, l1_reg: 4.6966e+02, l2_reg: 1.4681e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 957,\n",
      " njev: 957,\n",
      " nit: 901\n",
      "2020/09/13, 11:24:10, Batch: 73, BFGS Loss: 1.9293e-01, c1: 1.1379e-02, c2: 2.9090e-02, d1: 9.3147e-03, d2: 1.1294e-04, l1_reg: 4.8298e+02, l2_reg: 1.5585e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 374,\n",
      " njev: 374,\n",
      " nit: 285\n",
      "2020/09/13, 11:24:24, Batch: 74, BFGS Loss: 1.8521e-01, c1: 9.3458e-03, c2: 2.7753e-02, d1: 9.1672e-03, d2: 9.3125e-05, l1_reg: 4.8353e+02, l2_reg: 1.5681e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 290,\n",
      " njev: 290,\n",
      " nit: 195\n",
      "2020/09/13, 11:24:36, Batch: 75, BFGS Loss: 1.8238e-01, c1: 8.6824e-03, c2: 2.7323e-02, d1: 9.0829e-03, d2: 8.8940e-05, l1_reg: 4.8411e+02, l2_reg: 1.5733e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 276,\n",
      " njev: 276,\n",
      " nit: 156\n",
      "2020/09/13, 11:24:48, Batch: 76, BFGS Loss: 1.8098e-01, c1: 8.4644e-03, c2: 2.7102e-02, d1: 9.0329e-03, d2: 8.7690e-05, l1_reg: 4.8437e+02, l2_reg: 1.5761e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 245,\n",
      " njev: 245,\n",
      " nit: 137\n",
      "2020/09/13, 11:24:59, Batch: 77, BFGS Loss: 1.8009e-01, c1: 8.1924e-03, c2: 2.7037e-02, d1: 8.9956e-03, d2: 8.2719e-05, l1_reg: 4.8456e+02, l2_reg: 1.5779e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 22,\n",
      " njev: 22,\n",
      " nit: 2\n",
      "2020/09/13, 11:25:08, Batch: 78, BFGS Loss: 1.9007e-01, c1: 1.0210e-02, c2: 2.7369e-02, d1: 9.2381e-03, d2: 5.3698e-04, l1_reg: 4.8456e+02, l2_reg: 1.5779e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 251,\n",
      " njev: 251,\n",
      " nit: 163\n",
      "2020/09/13, 11:25:19, Batch: 79, BFGS Loss: 1.7880e-01, c1: 8.2757e-03, c2: 2.6814e-02, d1: 8.9144e-03, d2: 9.3158e-05, l1_reg: 4.8505e+02, l2_reg: 1.5828e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 314,\n",
      " njev: 314,\n",
      " nit: 208\n",
      "2020/09/13, 11:25:31, Batch: 80, BFGS Loss: 1.7679e-01, c1: 8.1168e-03, c2: 2.6566e-02, d1: 8.8009e-03, d2: 9.5610e-05, l1_reg: 4.8578e+02, l2_reg: 1.5899e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 315,\n",
      " njev: 315,\n",
      " nit: 218\n",
      "2020/09/13, 11:25:44, Batch: 81, BFGS Loss: 1.7460e-01, c1: 7.7956e-03, c2: 2.6258e-02, d1: 8.7072e-03, d2: 9.5314e-05, l1_reg: 4.8639e+02, l2_reg: 1.5961e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 282,\n",
      " njev: 282,\n",
      " nit: 176\n",
      "2020/09/13, 11:25:56, Batch: 82, BFGS Loss: 1.7330e-01, c1: 7.4679e-03, c2: 2.6156e-02, d1: 8.6538e-03, d2: 8.1489e-05, l1_reg: 4.8667e+02, l2_reg: 1.5988e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 65,\n",
      " njev: 65,\n",
      " nit: 9\n",
      "2020/09/13, 11:26:05, Batch: 83, BFGS Loss: 1.7583e-01, c1: 8.4693e-03, c2: 2.6167e-02, d1: 8.6597e-03, d2: 2.2518e-04, l1_reg: 4.8667e+02, l2_reg: 1.5988e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 283,\n",
      " njev: 283,\n",
      " nit: 188\n",
      "2020/09/13, 11:26:16, Batch: 84, BFGS Loss: 1.7222e-01, c1: 7.4942e-03, c2: 2.5942e-02, d1: 8.6029e-03, d2: 8.6701e-05, l1_reg: 4.8712e+02, l2_reg: 1.6034e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 299,\n",
      " njev: 299,\n",
      " nit: 187\n",
      "2020/09/13, 11:26:28, Batch: 85, BFGS Loss: 1.7109e-01, c1: 7.2759e-03, c2: 2.5832e-02, d1: 8.5493e-03, d2: 8.1736e-05, l1_reg: 4.8744e+02, l2_reg: 1.6065e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 60,\n",
      " njev: 60,\n",
      " nit: 8\n",
      "2020/09/13, 11:26:37, Batch: 86, BFGS Loss: 1.7311e-01, c1: 8.7392e-03, c2: 2.5870e-02, d1: 8.5630e-03, d2: 1.1269e-04, l1_reg: 4.8744e+02, l2_reg: 1.6065e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 277,\n",
      " njev: 277,\n",
      " nit: 169\n",
      "2020/09/13, 11:26:48, Batch: 87, BFGS Loss: 1.7029e-01, c1: 7.2167e-03, c2: 2.5711e-02, d1: 8.5141e-03, d2: 7.9613e-05, l1_reg: 4.8772e+02, l2_reg: 1.6094e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 247,\n",
      " njev: 247,\n",
      " nit: 115\n",
      "2020/09/13, 11:26:59, Batch: 88, BFGS Loss: 1.6986e-01, c1: 7.0796e-03, c2: 2.5672e-02, d1: 8.4976e-03, d2: 7.8225e-05, l1_reg: 4.8781e+02, l2_reg: 1.6102e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 36,\n",
      " njev: 36,\n",
      " nit: 4\n",
      "2020/09/13, 11:27:08, Batch: 89, BFGS Loss: 1.7305e-01, c1: 7.9156e-03, c2: 2.5743e-02, d1: 8.5625e-03, d2: 2.2685e-04, l1_reg: 4.8781e+02, l2_reg: 1.6102e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 132,\n",
      " njev: 132,\n",
      " nit: 39\n",
      "2020/09/13, 11:27:17, Batch: 90, BFGS Loss: 1.6993e-01, c1: 7.1352e-03, c2: 2.5672e-02, d1: 8.4982e-03, d2: 7.8920e-05, l1_reg: 4.8781e+02, l2_reg: 1.6102e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 178,\n",
      " njev: 178,\n",
      " nit: 45\n",
      "2020/09/13, 11:27:27, Batch: 91, BFGS Loss: 1.6993e-01, c1: 7.1548e-03, c2: 2.5667e-02, d1: 8.4989e-03, d2: 7.8202e-05, l1_reg: 4.8781e+02, l2_reg: 1.6103e+02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 140,\n",
      " njev: 140,\n",
      " nit: 28\n",
      "2020/09/13, 11:27:37, Batch: 92, BFGS Loss: 1.7004e-01, c1: 7.2340e-03, c2: 2.5676e-02, d1: 8.4965e-03, d2: 8.0288e-05, l1_reg: 4.8781e+02, l2_reg: 1.6103e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 139,\n",
      " njev: 139,\n",
      " nit: 33\n",
      "2020/09/13, 11:27:45, Batch: 93, BFGS Loss: 1.6983e-01, c1: 7.0679e-03, c2: 2.5665e-02, d1: 8.4984e-03, d2: 7.7367e-05, l1_reg: 4.8781e+02, l2_reg: 1.6103e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 93,\n",
      " njev: 93,\n",
      " nit: 12\n",
      "2020/09/13, 11:27:55, Batch: 94, BFGS Loss: 1.7655e-01, c1: 1.0487e-02, c2: 2.5952e-02, d1: 8.5179e-03, d2: 3.0224e-04, l1_reg: 4.8782e+02, l2_reg: 1.6103e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 111,\n",
      " njev: 111,\n",
      " nit: 35\n",
      "2020/09/13, 11:28:04, Batch: 95, BFGS Loss: 1.6988e-01, c1: 7.0993e-03, c2: 2.5666e-02, d1: 8.4983e-03, d2: 7.9250e-05, l1_reg: 4.8781e+02, l2_reg: 1.6103e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 132,\n",
      " njev: 132,\n",
      " nit: 29\n",
      "2020/09/13, 11:28:14, Batch: 96, BFGS Loss: 1.6991e-01, c1: 7.1405e-03, c2: 2.5663e-02, d1: 8.4986e-03, d2: 7.8986e-05, l1_reg: 4.8782e+02, l2_reg: 1.6103e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 112,\n",
      " njev: 112,\n",
      " nit: 21\n",
      "2020/09/13, 11:28:23, Batch: 97, BFGS Loss: 1.6996e-01, c1: 7.1787e-03, c2: 2.5667e-02, d1: 8.4982e-03, d2: 7.8903e-05, l1_reg: 4.8782e+02, l2_reg: 1.6103e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 105,\n",
      " njev: 105,\n",
      " nit: 19\n",
      "2020/09/13, 11:28:32, Batch: 98, BFGS Loss: 1.6995e-01, c1: 7.1574e-03, c2: 2.5666e-02, d1: 8.4996e-03, d2: 7.8990e-05, l1_reg: 4.8782e+02, l2_reg: 1.6103e+02\n",
      "Success: False,\n",
      " Status: 3,\n",
      " Message: status meaning: 0=converged, 1=max BFGS iters reached, 3=zoom failed, 4=saddle point reached, 5=max line search iters reached, -1=undefined,\n",
      " nfev: 126,\n",
      " njev: 126,\n",
      " nit: 27\n",
      "2020/09/13, 11:28:42, Batch: 99, BFGS Loss: 1.7010e-01, c1: 7.2346e-03, c2: 2.5688e-02, d1: 8.4985e-03, d2: 8.0748e-05, l1_reg: 4.8782e+02, l2_reg: 1.6103e+02\n"
     ]
    }
   ],
   "source": [
    "n_batchs = 100\n",
    "scale = 1e-4\n",
    "options = {\n",
    "\t\"maxiter\": 100000,\n",
    "\t\"line_search_maxiter\": 1000,\n",
    "}\n",
    "params_ = agent.train_bfgs(n_batchs, batch_fn, options, config.loss_names, config.log_file, scale = scale)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/13, 12:46:00, Iteration: 101000, Train Loss: 2.8850e-01, c1: 4.7401e-02, c2: 4.0221e-02, d1: 8.4406e-03, d2: 3.6021e-03, l1_reg: 4.9093e+02, l2_reg: 1.6418e+02\n",
      "2020/09/13, 12:46:10, Iteration: 102000, Train Loss: 3.2784e-01, c1: 5.3434e-02, c2: 4.2903e-02, d1: 8.7017e-03, d2: 5.8676e-03, l1_reg: 4.9160e+02, l2_reg: 1.6492e+02\n",
      "2020/09/13, 12:46:19, Iteration: 103000, Train Loss: 2.5657e-01, c1: 5.6619e-02, c2: 3.5396e-02, d1: 8.6155e-03, d2: 7.6000e-04, l1_reg: 4.9203e+02, l2_reg: 1.6545e+02\n",
      "2020/09/13, 12:46:29, Iteration: 104000, Train Loss: 2.8279e-01, c1: 5.1698e-02, c2: 3.9256e-02, d1: 8.1498e-03, d2: 3.1815e-03, l1_reg: 4.9254e+02, l2_reg: 1.6606e+02\n",
      "2020/09/13, 12:46:38, Iteration: 105000, Train Loss: 3.2150e-01, c1: 5.1733e-02, c2: 4.1406e-02, d1: 8.6383e-03, d2: 5.9161e-03, l1_reg: 4.9292e+02, l2_reg: 1.6656e+02\n",
      "2020/09/13, 12:46:48, Iteration: 106000, Train Loss: 3.4022e-01, c1: 9.6439e-02, c2: 4.1451e-02, d1: 9.3747e-03, d2: 2.5677e-03, l1_reg: 4.9330e+02, l2_reg: 1.6708e+02\n",
      "2020/09/13, 12:46:57, Iteration: 107000, Train Loss: 2.8412e-01, c1: 5.6906e-02, c2: 3.6742e-02, d1: 8.1788e-03, d2: 3.5191e-03, l1_reg: 4.9372e+02, l2_reg: 1.6762e+02\n",
      "2020/09/13, 12:47:07, Iteration: 108000, Train Loss: 2.7540e-01, c1: 5.9883e-02, c2: 3.6805e-02, d1: 8.4275e-03, d2: 2.0824e-03, l1_reg: 4.9387e+02, l2_reg: 1.6804e+02\n",
      "2020/09/13, 12:47:17, Iteration: 109000, Train Loss: 2.8935e-01, c1: 4.6779e-02, c2: 3.8230e-02, d1: 8.1141e-03, d2: 4.6731e-03, l1_reg: 4.9420e+02, l2_reg: 1.6851e+02\n",
      "2020/09/13, 12:47:26, Iteration: 110000, Train Loss: 2.6833e-01, c1: 4.6409e-02, c2: 3.5849e-02, d1: 8.3387e-03, d2: 3.0977e-03, l1_reg: 4.9454e+02, l2_reg: 1.6901e+02\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3.8/site-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  return array(a, dtype, copy=False, order=order)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/13, 12:47:36, Iteration: 111000, Train Loss: 2.6851e-01, c1: 4.6006e-02, c2: 3.9032e-02, d1: 8.9233e-03, d2: 1.6170e-03, l1_reg: 4.9474e+02, l2_reg: 1.6942e+02\n",
      "2020/09/13, 12:47:46, Iteration: 112000, Train Loss: 2.7145e-01, c1: 5.1340e-02, c2: 3.4882e-02, d1: 8.9950e-03, d2: 2.5511e-03, l1_reg: 4.9495e+02, l2_reg: 1.6980e+02\n",
      "2020/09/13, 12:47:55, Iteration: 113000, Train Loss: 2.4701e-01, c1: 3.8380e-02, c2: 3.4226e-02, d1: 8.2875e-03, d2: 2.3067e-03, l1_reg: 4.9539e+02, l2_reg: 1.7030e+02\n",
      "2020/09/13, 12:48:05, Iteration: 114000, Train Loss: 2.6437e-01, c1: 4.3930e-02, c2: 3.3228e-02, d1: 8.4470e-03, d2: 3.6279e-03, l1_reg: 4.9571e+02, l2_reg: 1.7072e+02\n",
      "2020/09/13, 12:48:15, Iteration: 115000, Train Loss: 2.7812e-01, c1: 4.9754e-02, c2: 3.5179e-02, d1: 8.3909e-03, d2: 3.8912e-03, l1_reg: 4.9601e+02, l2_reg: 1.7109e+02\n",
      "2020/09/13, 12:48:24, Iteration: 116000, Train Loss: 2.7627e-01, c1: 4.8496e-02, c2: 3.6361e-02, d1: 8.7170e-03, d2: 3.1511e-03, l1_reg: 4.9661e+02, l2_reg: 1.7159e+02\n",
      "2020/09/13, 12:48:34, Iteration: 117000, Train Loss: 2.7071e-01, c1: 5.9730e-02, c2: 3.6518e-02, d1: 8.5853e-03, d2: 1.5569e-03, l1_reg: 4.9683e+02, l2_reg: 1.7191e+02\n",
      "2020/09/13, 12:48:43, Iteration: 118000, Train Loss: 2.7176e-01, c1: 5.8140e-02, c2: 3.6388e-02, d1: 9.0875e-03, d2: 1.3575e-03, l1_reg: 4.9745e+02, l2_reg: 1.7235e+02\n",
      "2020/09/13, 12:48:53, Iteration: 119000, Train Loss: 2.7604e-01, c1: 4.9048e-02, c2: 3.6752e-02, d1: 9.2756e-03, d2: 2.3978e-03, l1_reg: 4.9788e+02, l2_reg: 1.7275e+02\n",
      "2020/09/13, 12:49:02, Iteration: 120000, Train Loss: 2.6801e-01, c1: 6.3453e-02, c2: 3.1704e-02, d1: 8.5919e-03, d2: 2.3523e-03, l1_reg: 4.9841e+02, l2_reg: 1.7322e+02\n",
      "2020/09/13, 12:49:12, Iteration: 121000, Train Loss: 2.4550e-01, c1: 3.7881e-02, c2: 3.2086e-02, d1: 8.5683e-03, d2: 2.5677e-03, l1_reg: 4.9873e+02, l2_reg: 1.7360e+02\n",
      "2020/09/13, 12:49:21, Iteration: 122000, Train Loss: 2.4001e-01, c1: 3.5952e-02, c2: 3.1113e-02, d1: 8.4459e-03, d2: 2.6248e-03, l1_reg: 4.9929e+02, l2_reg: 1.7403e+02\n",
      "2020/09/13, 12:49:31, Iteration: 123000, Train Loss: 2.4870e-01, c1: 4.1328e-02, c2: 3.4191e-02, d1: 8.3040e-03, d2: 2.1750e-03, l1_reg: 4.9985e+02, l2_reg: 1.7447e+02\n",
      "2020/09/13, 12:49:41, Iteration: 124000, Train Loss: 2.5488e-01, c1: 4.8927e-02, c2: 3.2010e-02, d1: 8.5463e-03, d2: 2.4458e-03, l1_reg: 5.0020e+02, l2_reg: 1.7485e+02\n",
      "2020/09/13, 12:49:50, Iteration: 125000, Train Loss: 2.6161e-01, c1: 4.0679e-02, c2: 3.3801e-02, d1: 9.4373e-03, d2: 2.5145e-03, l1_reg: 5.0087e+02, l2_reg: 1.7532e+02\n",
      "2020/09/13, 12:50:00, Iteration: 126000, Train Loss: 2.7461e-01, c1: 4.6415e-02, c2: 3.6982e-02, d1: 9.1251e-03, d2: 2.5986e-03, l1_reg: 5.0123e+02, l2_reg: 1.7571e+02\n",
      "2020/09/13, 12:50:09, Iteration: 127000, Train Loss: 2.5951e-01, c1: 4.8164e-02, c2: 3.1504e-02, d1: 8.6522e-03, d2: 3.0304e-03, l1_reg: 5.0185e+02, l2_reg: 1.7619e+02\n",
      "2020/09/13, 12:50:19, Iteration: 128000, Train Loss: 2.3777e-01, c1: 4.3094e-02, c2: 3.1009e-02, d1: 8.2831e-03, d2: 1.8814e-03, l1_reg: 5.0243e+02, l2_reg: 1.7664e+02\n",
      "2020/09/13, 12:50:28, Iteration: 129000, Train Loss: 2.4760e-01, c1: 4.3118e-02, c2: 3.0764e-02, d1: 8.5669e-03, d2: 2.6514e-03, l1_reg: 5.0284e+02, l2_reg: 1.7703e+02\n",
      "2020/09/13, 12:50:38, Iteration: 130000, Train Loss: 2.4668e-01, c1: 4.3975e-02, c2: 3.3453e-02, d1: 8.8380e-03, d2: 1.3957e-03, l1_reg: 5.0336e+02, l2_reg: 1.7746e+02\n",
      "2020/09/13, 12:50:48, Iteration: 131000, Train Loss: 2.4979e-01, c1: 4.4981e-02, c2: 3.2603e-02, d1: 8.3422e-03, d2: 2.3573e-03, l1_reg: 5.0399e+02, l2_reg: 1.7794e+02\n",
      "2020/09/13, 12:50:57, Iteration: 132000, Train Loss: 2.7851e-01, c1: 4.8980e-02, c2: 3.4771e-02, d1: 8.8123e-03, d2: 3.7083e-03, l1_reg: 5.0445e+02, l2_reg: 1.7835e+02\n",
      "2020/09/13, 12:51:07, Iteration: 133000, Train Loss: 2.7886e-01, c1: 5.5021e-02, c2: 3.3774e-02, d1: 8.6825e-03, d2: 3.5682e-03, l1_reg: 5.0501e+02, l2_reg: 1.7879e+02\n",
      "2020/09/13, 12:51:16, Iteration: 134000, Train Loss: 2.6607e-01, c1: 5.8763e-02, c2: 3.0999e-02, d1: 8.2452e-03, d2: 3.1854e-03, l1_reg: 5.0556e+02, l2_reg: 1.7924e+02\n",
      "2020/09/13, 12:51:26, Iteration: 135000, Train Loss: 2.6977e-01, c1: 5.6700e-02, c2: 3.1587e-02, d1: 8.5651e-03, d2: 3.2654e-03, l1_reg: 5.0615e+02, l2_reg: 1.7970e+02\n",
      "2020/09/13, 12:51:35, Iteration: 136000, Train Loss: 2.7884e-01, c1: 5.1636e-02, c2: 3.2758e-02, d1: 9.3943e-03, d2: 3.4981e-03, l1_reg: 5.0669e+02, l2_reg: 1.8012e+02\n",
      "2020/09/13, 12:51:45, Iteration: 137000, Train Loss: 2.3727e-01, c1: 3.2004e-02, c2: 3.0004e-02, d1: 8.5843e-03, d2: 2.9404e-03, l1_reg: 5.0719e+02, l2_reg: 1.8059e+02\n",
      "2020/09/13, 12:51:55, Iteration: 138000, Train Loss: 2.4953e-01, c1: 5.4342e-02, c2: 3.1361e-02, d1: 8.4925e-03, d2: 1.6172e-03, l1_reg: 5.0786e+02, l2_reg: 1.8109e+02\n",
      "2020/09/13, 12:52:04, Iteration: 139000, Train Loss: 2.4037e-01, c1: 4.4926e-02, c2: 2.9418e-02, d1: 8.6763e-03, d2: 2.0423e-03, l1_reg: 5.0841e+02, l2_reg: 1.8152e+02\n",
      "2020/09/13, 12:52:14, Iteration: 140000, Train Loss: 2.4892e-01, c1: 4.2987e-02, c2: 3.2046e-02, d1: 8.1443e-03, d2: 2.8346e-03, l1_reg: 5.0902e+02, l2_reg: 1.8202e+02\n",
      "2020/09/13, 12:52:24, Iteration: 141000, Train Loss: 2.6366e-01, c1: 4.9206e-02, c2: 3.1361e-02, d1: 8.3900e-03, d2: 3.6464e-03, l1_reg: 5.0970e+02, l2_reg: 1.8252e+02\n",
      "2020/09/13, 12:52:33, Iteration: 142000, Train Loss: 2.9759e-01, c1: 6.8482e-02, c2: 3.6327e-02, d1: 9.3520e-03, d2: 2.6598e-03, l1_reg: 5.1018e+02, l2_reg: 1.8296e+02\n",
      "2020/09/13, 12:52:43, Iteration: 143000, Train Loss: 2.6227e-01, c1: 4.0878e-02, c2: 3.5498e-02, d1: 8.9670e-03, d2: 2.5216e-03, l1_reg: 5.1079e+02, l2_reg: 1.8345e+02\n",
      "2020/09/13, 12:52:52, Iteration: 144000, Train Loss: 2.3314e-01, c1: 4.8679e-02, c2: 3.0340e-02, d1: 8.2509e-03, d2: 1.0927e-03, l1_reg: 5.1131e+02, l2_reg: 1.8389e+02\n",
      "2020/09/13, 12:53:02, Iteration: 145000, Train Loss: 2.5076e-01, c1: 4.9263e-02, c2: 3.3389e-02, d1: 9.0043e-03, d2: 1.1280e-03, l1_reg: 5.1197e+02, l2_reg: 1.8440e+02\n",
      "2020/09/13, 12:53:11, Iteration: 146000, Train Loss: 2.7116e-01, c1: 6.0604e-02, c2: 3.2208e-02, d1: 9.2934e-03, d2: 2.0993e-03, l1_reg: 5.1248e+02, l2_reg: 1.8486e+02\n",
      "2020/09/13, 12:53:21, Iteration: 147000, Train Loss: 2.6925e-01, c1: 4.6857e-02, c2: 3.1881e-02, d1: 9.2349e-03, d2: 3.4395e-03, l1_reg: 5.1307e+02, l2_reg: 1.8535e+02\n",
      "2020/09/13, 12:53:30, Iteration: 148000, Train Loss: 2.5038e-01, c1: 4.8753e-02, c2: 3.1179e-02, d1: 9.0404e-03, d2: 1.7676e-03, l1_reg: 5.1369e+02, l2_reg: 1.8583e+02\n",
      "2020/09/13, 12:53:40, Iteration: 149000, Train Loss: 2.6774e-01, c1: 4.0187e-02, c2: 3.0472e-02, d1: 9.0854e-03, d2: 4.5275e-03, l1_reg: 5.1426e+02, l2_reg: 1.8628e+02\n",
      "2020/09/13, 12:53:49, Iteration: 150000, Train Loss: 2.9003e-01, c1: 7.0454e-02, c2: 3.3020e-02, d1: 8.0521e-03, d2: 3.9985e-03, l1_reg: 5.1478e+02, l2_reg: 1.8675e+02\n",
      "2020/09/13, 12:53:59, Iteration: 151000, Train Loss: 2.3256e-01, c1: 4.4086e-02, c2: 2.9799e-02, d1: 8.3254e-03, d2: 1.5815e-03, l1_reg: 5.1527e+02, l2_reg: 1.8719e+02\n",
      "2020/09/13, 12:54:09, Iteration: 152000, Train Loss: 2.3512e-01, c1: 4.5484e-02, c2: 2.8274e-02, d1: 8.3219e-03, d2: 2.1591e-03, l1_reg: 5.1581e+02, l2_reg: 1.8768e+02\n",
      "2020/09/13, 12:54:18, Iteration: 153000, Train Loss: 2.3373e-01, c1: 3.5742e-02, c2: 3.2010e-02, d1: 9.2312e-03, d2: 9.6435e-04, l1_reg: 5.1632e+02, l2_reg: 1.8813e+02\n",
      "2020/09/13, 12:54:28, Iteration: 154000, Train Loss: 2.5138e-01, c1: 4.7683e-02, c2: 3.2671e-02, d1: 8.9296e-03, d2: 1.6379e-03, l1_reg: 5.1682e+02, l2_reg: 1.8858e+02\n",
      "2020/09/13, 12:54:37, Iteration: 155000, Train Loss: 2.4925e-01, c1: 4.3946e-02, c2: 3.1511e-02, d1: 8.5598e-03, d2: 2.5170e-03, l1_reg: 5.1731e+02, l2_reg: 1.8903e+02\n",
      "2020/09/13, 12:54:47, Iteration: 156000, Train Loss: 2.3979e-01, c1: 4.8506e-02, c2: 2.9546e-02, d1: 8.4336e-03, d2: 1.8308e-03, l1_reg: 5.1774e+02, l2_reg: 1.8944e+02\n",
      "2020/09/13, 12:54:56, Iteration: 157000, Train Loss: 2.4130e-01, c1: 5.0308e-02, c2: 3.0752e-02, d1: 7.9631e-03, d2: 1.9094e-03, l1_reg: 5.1841e+02, l2_reg: 1.8996e+02\n",
      "2020/09/13, 12:55:06, Iteration: 158000, Train Loss: 2.5454e-01, c1: 4.8742e-02, c2: 2.9010e-02, d1: 9.0354e-03, d2: 2.8402e-03, l1_reg: 5.1900e+02, l2_reg: 1.9038e+02\n",
      "2020/09/13, 12:55:16, Iteration: 159000, Train Loss: 2.7545e-01, c1: 5.7009e-02, c2: 3.2724e-02, d1: 9.1831e-03, d2: 2.8436e-03, l1_reg: 5.1926e+02, l2_reg: 1.9085e+02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020/09/13, 12:55:25, Iteration: 160000, Train Loss: 2.4897e-01, c1: 5.5189e-02, c2: 3.0446e-02, d1: 8.1033e-03, d2: 2.1406e-03, l1_reg: 5.1964e+02, l2_reg: 1.9128e+02\n",
      "2020/09/13, 12:55:35, Iteration: 161000, Train Loss: 2.3765e-01, c1: 5.3094e-02, c2: 2.8957e-02, d1: 8.2500e-03, d2: 1.5180e-03, l1_reg: 5.2004e+02, l2_reg: 1.9171e+02\n",
      "2020/09/13, 12:55:44, Iteration: 162000, Train Loss: 2.4139e-01, c1: 5.1529e-02, c2: 3.0118e-02, d1: 8.1566e-03, d2: 1.7930e-03, l1_reg: 5.2037e+02, l2_reg: 1.9213e+02\n",
      "2020/09/13, 12:55:54, Iteration: 163000, Train Loss: 2.4360e-01, c1: 5.2115e-02, c2: 3.1214e-02, d1: 8.0863e-03, d2: 1.6976e-03, l1_reg: 5.2063e+02, l2_reg: 1.9255e+02\n",
      "2020/09/13, 12:56:04, Iteration: 164000, Train Loss: 2.6132e-01, c1: 5.1938e-02, c2: 3.2125e-02, d1: 9.0937e-03, d2: 2.2065e-03, l1_reg: 5.2085e+02, l2_reg: 1.9293e+02\n",
      "2020/09/13, 12:56:13, Iteration: 165000, Train Loss: 2.2484e-01, c1: 3.9285e-02, c2: 2.8568e-02, d1: 8.0322e-03, d2: 1.9525e-03, l1_reg: 5.2105e+02, l2_reg: 1.9337e+02\n",
      "2020/09/13, 12:56:23, Iteration: 166000, Train Loss: 2.4822e-01, c1: 4.7491e-02, c2: 2.9498e-02, d1: 8.2154e-03, d2: 3.0071e-03, l1_reg: 5.2123e+02, l2_reg: 1.9375e+02\n",
      "2020/09/13, 12:56:32, Iteration: 167000, Train Loss: 2.4896e-01, c1: 4.1724e-02, c2: 3.0332e-02, d1: 8.5893e-03, d2: 3.0335e-03, l1_reg: 5.2163e+02, l2_reg: 1.9421e+02\n",
      "2020/09/13, 12:56:42, Iteration: 168000, Train Loss: 2.1310e-01, c1: 4.0696e-02, c2: 2.7416e-02, d1: 8.4244e-03, d2: 5.9061e-04, l1_reg: 5.2200e+02, l2_reg: 1.9463e+02\n",
      "2020/09/13, 12:56:51, Iteration: 169000, Train Loss: 2.3079e-01, c1: 3.6891e-02, c2: 2.8937e-02, d1: 8.7070e-03, d2: 2.0006e-03, l1_reg: 5.2242e+02, l2_reg: 1.9506e+02\n",
      "2020/09/13, 12:57:01, Iteration: 170000, Train Loss: 2.6989e-01, c1: 4.6568e-02, c2: 3.2437e-02, d1: 8.8589e-03, d2: 3.7410e-03, l1_reg: 5.2280e+02, l2_reg: 1.9548e+02\n",
      "2020/09/13, 12:57:11, Iteration: 171000, Train Loss: 2.2372e-01, c1: 4.0722e-02, c2: 2.7179e-02, d1: 8.4084e-03, d2: 1.7366e-03, l1_reg: 5.2318e+02, l2_reg: 1.9587e+02\n",
      "2020/09/13, 12:57:20, Iteration: 172000, Train Loss: 2.7197e-01, c1: 4.4255e-02, c2: 3.4352e-02, d1: 9.0681e-03, d2: 3.3967e-03, l1_reg: 5.2368e+02, l2_reg: 1.9634e+02\n",
      "2020/09/13, 12:57:30, Iteration: 173000, Train Loss: 2.3922e-01, c1: 5.1337e-02, c2: 2.8743e-02, d1: 7.8537e-03, d2: 2.3109e-03, l1_reg: 5.2400e+02, l2_reg: 1.9672e+02\n",
      "2020/09/13, 12:57:40, Iteration: 174000, Train Loss: 2.2789e-01, c1: 4.6898e-02, c2: 2.8061e-02, d1: 7.7542e-03, d2: 1.9263e-03, l1_reg: 5.2451e+02, l2_reg: 1.9718e+02\n",
      "2020/09/13, 12:57:49, Iteration: 175000, Train Loss: 2.1405e-01, c1: 3.6470e-02, c2: 2.6453e-02, d1: 8.1026e-03, d2: 1.7187e-03, l1_reg: 5.2491e+02, l2_reg: 1.9756e+02\n",
      "2020/09/13, 12:57:59, Iteration: 176000, Train Loss: 2.3133e-01, c1: 4.5291e-02, c2: 2.7216e-02, d1: 9.0506e-03, d2: 1.3881e-03, l1_reg: 5.2527e+02, l2_reg: 1.9797e+02\n",
      "2020/09/13, 12:58:08, Iteration: 177000, Train Loss: 2.6696e-01, c1: 5.3298e-02, c2: 3.2609e-02, d1: 8.7561e-03, d2: 2.8268e-03, l1_reg: 5.2573e+02, l2_reg: 1.9842e+02\n",
      "2020/09/13, 12:58:18, Iteration: 178000, Train Loss: 2.0844e-01, c1: 3.9451e-02, c2: 2.5510e-02, d1: 7.6517e-03, d2: 1.5934e-03, l1_reg: 5.2613e+02, l2_reg: 1.9880e+02\n",
      "2020/09/13, 12:58:28, Iteration: 179000, Train Loss: 2.3192e-01, c1: 4.3387e-02, c2: 2.7119e-02, d1: 8.4850e-03, d2: 2.2313e-03, l1_reg: 5.2659e+02, l2_reg: 1.9923e+02\n",
      "2020/09/13, 12:58:37, Iteration: 180000, Train Loss: 2.3779e-01, c1: 5.3036e-02, c2: 2.6616e-02, d1: 8.4525e-03, d2: 2.0370e-03, l1_reg: 5.2686e+02, l2_reg: 1.9957e+02\n",
      "2020/09/13, 12:58:47, Iteration: 181000, Train Loss: 2.2061e-01, c1: 5.2106e-02, c2: 2.5639e-02, d1: 7.7371e-03, d2: 1.4210e-03, l1_reg: 5.2731e+02, l2_reg: 2.0000e+02\n",
      "2020/09/13, 12:58:56, Iteration: 182000, Train Loss: 2.1775e-01, c1: 4.2772e-02, c2: 2.7159e-02, d1: 8.0393e-03, d2: 1.3098e-03, l1_reg: 5.2776e+02, l2_reg: 2.0041e+02\n",
      "2020/09/13, 12:59:06, Iteration: 183000, Train Loss: 2.5569e-01, c1: 3.8691e-02, c2: 3.0309e-02, d1: 8.2662e-03, d2: 4.3409e-03, l1_reg: 5.2826e+02, l2_reg: 2.0085e+02\n",
      "2020/09/13, 12:59:16, Iteration: 184000, Train Loss: 2.3725e-01, c1: 3.7798e-02, c2: 2.7912e-02, d1: 8.6268e-03, d2: 2.9440e-03, l1_reg: 5.2855e+02, l2_reg: 2.0122e+02\n",
      "2020/09/13, 12:59:25, Iteration: 185000, Train Loss: 2.3184e-01, c1: 4.0568e-02, c2: 2.9135e-02, d1: 7.8872e-03, d2: 2.4992e-03, l1_reg: 5.2902e+02, l2_reg: 2.0163e+02\n",
      "2020/09/13, 12:59:35, Iteration: 186000, Train Loss: 2.4312e-01, c1: 4.9956e-02, c2: 2.9073e-02, d1: 7.7303e-03, d2: 2.8635e-03, l1_reg: 5.2954e+02, l2_reg: 2.0209e+02\n",
      "2020/09/13, 12:59:44, Iteration: 187000, Train Loss: 2.4092e-01, c1: 5.6707e-02, c2: 2.8099e-02, d1: 8.7926e-03, d2: 1.1979e-03, l1_reg: 5.2992e+02, l2_reg: 2.0246e+02\n",
      "2020/09/13, 12:59:54, Iteration: 188000, Train Loss: 2.2190e-01, c1: 5.0034e-02, c2: 2.6374e-02, d1: 7.6425e-03, d2: 1.6313e-03, l1_reg: 5.3035e+02, l2_reg: 2.0282e+02\n",
      "2020/09/13, 13:00:03, Iteration: 189000, Train Loss: 2.1744e-01, c1: 3.9700e-02, c2: 2.6766e-02, d1: 7.9192e-03, d2: 1.8238e-03, l1_reg: 5.3083e+02, l2_reg: 2.0323e+02\n",
      "2020/09/13, 13:00:13, Iteration: 190000, Train Loss: 2.2036e-01, c1: 4.3013e-02, c2: 2.6429e-02, d1: 8.2370e-03, d2: 1.5679e-03, l1_reg: 5.3104e+02, l2_reg: 2.0357e+02\n",
      "2020/09/13, 13:00:22, Iteration: 191000, Train Loss: 2.6227e-01, c1: 5.4977e-02, c2: 3.3208e-02, d1: 9.0121e-03, d2: 1.7539e-03, l1_reg: 5.3155e+02, l2_reg: 2.0398e+02\n",
      "2020/09/13, 13:00:32, Iteration: 192000, Train Loss: 2.3217e-01, c1: 5.3149e-02, c2: 2.7083e-02, d1: 8.1317e-03, d2: 1.6453e-03, l1_reg: 5.3190e+02, l2_reg: 2.0437e+02\n",
      "2020/09/13, 13:00:42, Iteration: 193000, Train Loss: 2.5170e-01, c1: 6.0113e-02, c2: 2.7078e-02, d1: 8.4170e-03, d2: 2.6176e-03, l1_reg: 5.3242e+02, l2_reg: 2.0475e+02\n",
      "2020/09/13, 13:00:51, Iteration: 194000, Train Loss: 2.2642e-01, c1: 3.2720e-02, c2: 2.8072e-02, d1: 8.9840e-03, d2: 1.9639e-03, l1_reg: 5.3269e+02, l2_reg: 2.0512e+02\n",
      "2020/09/13, 13:01:01, Iteration: 195000, Train Loss: 2.5055e-01, c1: 5.5205e-02, c2: 2.9357e-02, d1: 8.2872e-03, d2: 2.4395e-03, l1_reg: 5.3308e+02, l2_reg: 2.0550e+02\n",
      "2020/09/13, 13:01:11, Iteration: 196000, Train Loss: 2.5815e-01, c1: 5.9473e-02, c2: 2.8856e-02, d1: 8.7671e-03, d2: 2.4429e-03, l1_reg: 5.3349e+02, l2_reg: 2.0588e+02\n",
      "2020/09/13, 13:01:20, Iteration: 197000, Train Loss: 2.1697e-01, c1: 3.2865e-02, c2: 2.6314e-02, d1: 8.8114e-03, d2: 1.7040e-03, l1_reg: 5.3396e+02, l2_reg: 2.0628e+02\n",
      "2020/09/13, 13:01:30, Iteration: 198000, Train Loss: 2.1917e-01, c1: 4.7819e-02, c2: 2.6534e-02, d1: 8.0368e-03, d2: 1.1371e-03, l1_reg: 5.3429e+02, l2_reg: 2.0661e+02\n",
      "2020/09/13, 13:01:39, Iteration: 199000, Train Loss: 2.1953e-01, c1: 4.7681e-02, c2: 2.6593e-02, d1: 7.6945e-03, d2: 1.5116e-03, l1_reg: 5.3468e+02, l2_reg: 2.0699e+02\n",
      "2020/09/13, 13:01:49, Iteration: 200000, Train Loss: 2.4050e-01, c1: 5.2653e-02, c2: 2.9702e-02, d1: 7.5124e-03, d2: 2.3611e-03, l1_reg: 5.3506e+02, l2_reg: 2.0736e+02\n"
     ]
    }
   ],
   "source": [
    "agent.compile(config.optimizer, config.lr, params_init = params_)\n",
    "\n",
    "config.iterations = 100000\n",
    "config.print_every = 1000\n",
    "agent.train(config.iterations, batch_fn, evaluate_batch_fn, config.print_every, config.save_every, config.loss_names, config.log_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jax\n",
    "import jax.numpy as jnp\n",
    "import numpy as np\n",
    "\n",
    "import os, sys\n",
    "sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(\"./\")))))\n",
    "\n",
    "from loss import model\n",
    "from jaxmeta.data import load_params, tensor_grid\n",
    "from jaxmeta.loss import rmse\n",
    "\n",
    "import config\n",
    "from data import domain\n",
    "\n",
    "from scipy.io import loadmat\n",
    "data_true = loadmat(\"a_3_epsilon_1e-12.mat\")\n",
    "# data_true = loadmat(\"../Goldstein_Taylor/epsilon_1e-12_fine_mesh_snapshots.mat\")\n",
    "u_true, v_true, x_test, t_test = data_true[\"u_snapshots\"], data_true[\"v_snapshots\"], data_true[\"x\"], data_true[\"t\"].T\n",
    "xt_test = [tensor_grid([x_test, t_test_i]) for t_test_i in t_test]\n",
    "\n",
    "params_ = agent.params\n",
    "uv_pred = [model(params_, xt) for xt in xt_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "/* Put everything inside the global mpl namespace */\n",
       "/* global mpl */\n",
       "window.mpl = {};\n",
       "\n",
       "mpl.get_websocket_type = function () {\n",
       "    if (typeof WebSocket !== 'undefined') {\n",
       "        return WebSocket;\n",
       "    } else if (typeof MozWebSocket !== 'undefined') {\n",
       "        return MozWebSocket;\n",
       "    } else {\n",
       "        alert(\n",
       "            'Your browser does not have WebSocket support. ' +\n",
       "                'Please try Chrome, Safari or Firefox ≥ 6. ' +\n",
       "                'Firefox 4 and 5 are also supported but you ' +\n",
       "                'have to enable WebSockets in about:config.'\n",
       "        );\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n",
       "    this.id = figure_id;\n",
       "\n",
       "    this.ws = websocket;\n",
       "\n",
       "    this.supports_binary = this.ws.binaryType !== undefined;\n",
       "\n",
       "    if (!this.supports_binary) {\n",
       "        var warnings = document.getElementById('mpl-warnings');\n",
       "        if (warnings) {\n",
       "            warnings.style.display = 'block';\n",
       "            warnings.textContent =\n",
       "                'This browser does not support binary websocket messages. ' +\n",
       "                'Performance may be slow.';\n",
       "        }\n",
       "    }\n",
       "\n",
       "    this.imageObj = new Image();\n",
       "\n",
       "    this.context = undefined;\n",
       "    this.message = undefined;\n",
       "    this.canvas = undefined;\n",
       "    this.rubberband_canvas = undefined;\n",
       "    this.rubberband_context = undefined;\n",
       "    this.format_dropdown = undefined;\n",
       "\n",
       "    this.image_mode = 'full';\n",
       "\n",
       "    this.root = document.createElement('div');\n",
       "    this.root.setAttribute('style', 'display: inline-block');\n",
       "    this._root_extra_style(this.root);\n",
       "\n",
       "    parent_element.appendChild(this.root);\n",
       "\n",
       "    this._init_header(this);\n",
       "    this._init_canvas(this);\n",
       "    this._init_toolbar(this);\n",
       "\n",
       "    var fig = this;\n",
       "\n",
       "    this.waiting = false;\n",
       "\n",
       "    this.ws.onopen = function () {\n",
       "        fig.send_message('supports_binary', { value: fig.supports_binary });\n",
       "        fig.send_message('send_image_mode', {});\n",
       "        if (mpl.ratio !== 1) {\n",
       "            fig.send_message('set_dpi_ratio', { dpi_ratio: mpl.ratio });\n",
       "        }\n",
       "        fig.send_message('refresh', {});\n",
       "    };\n",
       "\n",
       "    this.imageObj.onload = function () {\n",
       "        if (fig.image_mode === 'full') {\n",
       "            // Full images could contain transparency (where diff images\n",
       "            // almost always do), so we need to clear the canvas so that\n",
       "            // there is no ghosting.\n",
       "            fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n",
       "        }\n",
       "        fig.context.drawImage(fig.imageObj, 0, 0);\n",
       "    };\n",
       "\n",
       "    this.imageObj.onunload = function () {\n",
       "        fig.ws.close();\n",
       "    };\n",
       "\n",
       "    this.ws.onmessage = this._make_on_message_function(this);\n",
       "\n",
       "    this.ondownload = ondownload;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._init_header = function () {\n",
       "    var titlebar = document.createElement('div');\n",
       "    titlebar.classList =\n",
       "        'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n",
       "    var titletext = document.createElement('div');\n",
       "    titletext.classList = 'ui-dialog-title';\n",
       "    titletext.setAttribute(\n",
       "        'style',\n",
       "        'width: 100%; text-align: center; padding: 3px;'\n",
       "    );\n",
       "    titlebar.appendChild(titletext);\n",
       "    this.root.appendChild(titlebar);\n",
       "    this.header = titletext;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n",
       "\n",
       "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n",
       "\n",
       "mpl.figure.prototype._init_canvas = function () {\n",
       "    var fig = this;\n",
       "\n",
       "    var canvas_div = (this.canvas_div = document.createElement('div'));\n",
       "    canvas_div.setAttribute(\n",
       "        'style',\n",
       "        'border: 1px solid #ddd;' +\n",
       "            'box-sizing: content-box;' +\n",
       "            'clear: both;' +\n",
       "            'min-height: 1px;' +\n",
       "            'min-width: 1px;' +\n",
       "            'outline: 0;' +\n",
       "            'overflow: hidden;' +\n",
       "            'position: relative;' +\n",
       "            'resize: both;'\n",
       "    );\n",
       "\n",
       "    function on_keyboard_event_closure(name) {\n",
       "        return function (event) {\n",
       "            return fig.key_event(event, name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    canvas_div.addEventListener(\n",
       "        'keydown',\n",
       "        on_keyboard_event_closure('key_press')\n",
       "    );\n",
       "    canvas_div.addEventListener(\n",
       "        'keyup',\n",
       "        on_keyboard_event_closure('key_release')\n",
       "    );\n",
       "\n",
       "    this._canvas_extra_style(canvas_div);\n",
       "    this.root.appendChild(canvas_div);\n",
       "\n",
       "    var canvas = (this.canvas = document.createElement('canvas'));\n",
       "    canvas.classList.add('mpl-canvas');\n",
       "    canvas.setAttribute('style', 'box-sizing: content-box;');\n",
       "\n",
       "    this.context = canvas.getContext('2d');\n",
       "\n",
       "    var backingStore =\n",
       "        this.context.backingStorePixelRatio ||\n",
       "        this.context.webkitBackingStorePixelRatio ||\n",
       "        this.context.mozBackingStorePixelRatio ||\n",
       "        this.context.msBackingStorePixelRatio ||\n",
       "        this.context.oBackingStorePixelRatio ||\n",
       "        this.context.backingStorePixelRatio ||\n",
       "        1;\n",
       "\n",
       "    mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n",
       "\n",
       "    var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n",
       "        'canvas'\n",
       "    ));\n",
       "    rubberband_canvas.setAttribute(\n",
       "        'style',\n",
       "        'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n",
       "    );\n",
       "\n",
       "    var resizeObserver = new ResizeObserver(function (entries) {\n",
       "        var nentries = entries.length;\n",
       "        for (var i = 0; i < nentries; i++) {\n",
       "            var entry = entries[i];\n",
       "            var width, height;\n",
       "            if (entry.contentBoxSize) {\n",
       "                width = entry.contentBoxSize.inlineSize;\n",
       "                height = entry.contentBoxSize.blockSize;\n",
       "            } else {\n",
       "                width = entry.contentRect.width;\n",
       "                height = entry.contentRect.height;\n",
       "            }\n",
       "\n",
       "            // Keep the size of the canvas and rubber band canvas in sync with\n",
       "            // the canvas container.\n",
       "            canvas.setAttribute('width', width * mpl.ratio);\n",
       "            canvas.setAttribute('height', height * mpl.ratio);\n",
       "            canvas.setAttribute(\n",
       "                'style',\n",
       "                'width: ' + width + 'px; height: ' + height + 'px;'\n",
       "            );\n",
       "\n",
       "            rubberband_canvas.setAttribute('width', width);\n",
       "            rubberband_canvas.setAttribute('height', height);\n",
       "\n",
       "            // And update the size in Python. We ignore the initial 0/0 size\n",
       "            // that occurs as the element is placed into the DOM, which should\n",
       "            // otherwise not happen due to the minimum size styling.\n",
       "            if (width != 0 && height != 0) {\n",
       "                fig.request_resize(width, height);\n",
       "            }\n",
       "        }\n",
       "    });\n",
       "    resizeObserver.observe(canvas_div);\n",
       "\n",
       "    function on_mouse_event_closure(name) {\n",
       "        return function (event) {\n",
       "            return fig.mouse_event(event, name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mousedown',\n",
       "        on_mouse_event_closure('button_press')\n",
       "    );\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mouseup',\n",
       "        on_mouse_event_closure('button_release')\n",
       "    );\n",
       "    // Throttle sequential mouse events to 1 every 20ms.\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mousemove',\n",
       "        on_mouse_event_closure('motion_notify')\n",
       "    );\n",
       "\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mouseenter',\n",
       "        on_mouse_event_closure('figure_enter')\n",
       "    );\n",
       "    rubberband_canvas.addEventListener(\n",
       "        'mouseleave',\n",
       "        on_mouse_event_closure('figure_leave')\n",
       "    );\n",
       "\n",
       "    canvas_div.addEventListener('wheel', function (event) {\n",
       "        if (event.deltaY < 0) {\n",
       "            event.step = 1;\n",
       "        } else {\n",
       "            event.step = -1;\n",
       "        }\n",
       "        on_mouse_event_closure('scroll')(event);\n",
       "    });\n",
       "\n",
       "    canvas_div.appendChild(canvas);\n",
       "    canvas_div.appendChild(rubberband_canvas);\n",
       "\n",
       "    this.rubberband_context = rubberband_canvas.getContext('2d');\n",
       "    this.rubberband_context.strokeStyle = '#000000';\n",
       "\n",
       "    this._resize_canvas = function (width, height, forward) {\n",
       "        if (forward) {\n",
       "            canvas_div.style.width = width + 'px';\n",
       "            canvas_div.style.height = height + 'px';\n",
       "        }\n",
       "    };\n",
       "\n",
       "    // Disable right mouse context menu.\n",
       "    this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n",
       "        return false;\n",
       "    });\n",
       "\n",
       "    function set_focus() {\n",
       "        canvas.focus();\n",
       "        canvas_div.focus();\n",
       "    }\n",
       "\n",
       "    window.setTimeout(set_focus, 100);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._init_toolbar = function () {\n",
       "    var fig = this;\n",
       "\n",
       "    var toolbar = document.createElement('div');\n",
       "    toolbar.classList = 'mpl-toolbar';\n",
       "    this.root.appendChild(toolbar);\n",
       "\n",
       "    function on_click_closure(name) {\n",
       "        return function (_event) {\n",
       "            return fig.toolbar_button_onclick(name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    function on_mouseover_closure(tooltip) {\n",
       "        return function (event) {\n",
       "            if (!event.currentTarget.disabled) {\n",
       "                return fig.toolbar_button_onmouseover(tooltip);\n",
       "            }\n",
       "        };\n",
       "    }\n",
       "\n",
       "    fig.buttons = {};\n",
       "    var buttonGroup = document.createElement('div');\n",
       "    buttonGroup.classList = 'mpl-button-group';\n",
       "    for (var toolbar_ind in mpl.toolbar_items) {\n",
       "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
       "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
       "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
       "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
       "\n",
       "        if (!name) {\n",
       "            /* Instead of a spacer, we start a new button group. */\n",
       "            if (buttonGroup.hasChildNodes()) {\n",
       "                toolbar.appendChild(buttonGroup);\n",
       "            }\n",
       "            buttonGroup = document.createElement('div');\n",
       "            buttonGroup.classList = 'mpl-button-group';\n",
       "            continue;\n",
       "        }\n",
       "\n",
       "        var button = (fig.buttons[name] = document.createElement('button'));\n",
       "        button.classList = 'mpl-widget';\n",
       "        button.setAttribute('role', 'button');\n",
       "        button.setAttribute('aria-disabled', 'false');\n",
       "        button.addEventListener('click', on_click_closure(method_name));\n",
       "        button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n",
       "\n",
       "        var icon_img = document.createElement('img');\n",
       "        icon_img.src = '_images/' + image + '.png';\n",
       "        icon_img.srcset = '_images/' + image + '_large.png 2x';\n",
       "        icon_img.alt = tooltip;\n",
       "        button.appendChild(icon_img);\n",
       "\n",
       "        buttonGroup.appendChild(button);\n",
       "    }\n",
       "\n",
       "    if (buttonGroup.hasChildNodes()) {\n",
       "        toolbar.appendChild(buttonGroup);\n",
       "    }\n",
       "\n",
       "    var fmt_picker = document.createElement('select');\n",
       "    fmt_picker.classList = 'mpl-widget';\n",
       "    toolbar.appendChild(fmt_picker);\n",
       "    this.format_dropdown = fmt_picker;\n",
       "\n",
       "    for (var ind in mpl.extensions) {\n",
       "        var fmt = mpl.extensions[ind];\n",
       "        var option = document.createElement('option');\n",
       "        option.selected = fmt === mpl.default_extension;\n",
       "        option.innerHTML = fmt;\n",
       "        fmt_picker.appendChild(option);\n",
       "    }\n",
       "\n",
       "    var status_bar = document.createElement('span');\n",
       "    status_bar.classList = 'mpl-message';\n",
       "    toolbar.appendChild(status_bar);\n",
       "    this.message = status_bar;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n",
       "    // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n",
       "    // which will in turn request a refresh of the image.\n",
       "    this.send_message('resize', { width: x_pixels, height: y_pixels });\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.send_message = function (type, properties) {\n",
       "    properties['type'] = type;\n",
       "    properties['figure_id'] = this.id;\n",
       "    this.ws.send(JSON.stringify(properties));\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.send_draw_message = function () {\n",
       "    if (!this.waiting) {\n",
       "        this.waiting = true;\n",
       "        this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_save = function (fig, _msg) {\n",
       "    var format_dropdown = fig.format_dropdown;\n",
       "    var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n",
       "    fig.ondownload(fig, format);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_resize = function (fig, msg) {\n",
       "    var size = msg['size'];\n",
       "    if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n",
       "        fig._resize_canvas(size[0], size[1], msg['forward']);\n",
       "        fig.send_message('refresh', {});\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n",
       "    var x0 = msg['x0'] / mpl.ratio;\n",
       "    var y0 = (fig.canvas.height - msg['y0']) / mpl.ratio;\n",
       "    var x1 = msg['x1'] / mpl.ratio;\n",
       "    var y1 = (fig.canvas.height - msg['y1']) / mpl.ratio;\n",
       "    x0 = Math.floor(x0) + 0.5;\n",
       "    y0 = Math.floor(y0) + 0.5;\n",
       "    x1 = Math.floor(x1) + 0.5;\n",
       "    y1 = Math.floor(y1) + 0.5;\n",
       "    var min_x = Math.min(x0, x1);\n",
       "    var min_y = Math.min(y0, y1);\n",
       "    var width = Math.abs(x1 - x0);\n",
       "    var height = Math.abs(y1 - y0);\n",
       "\n",
       "    fig.rubberband_context.clearRect(\n",
       "        0,\n",
       "        0,\n",
       "        fig.canvas.width / mpl.ratio,\n",
       "        fig.canvas.height / mpl.ratio\n",
       "    );\n",
       "\n",
       "    fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n",
       "    // Updates the figure title.\n",
       "    fig.header.textContent = msg['label'];\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n",
       "    var cursor = msg['cursor'];\n",
       "    switch (cursor) {\n",
       "        case 0:\n",
       "            cursor = 'pointer';\n",
       "            break;\n",
       "        case 1:\n",
       "            cursor = 'default';\n",
       "            break;\n",
       "        case 2:\n",
       "            cursor = 'crosshair';\n",
       "            break;\n",
       "        case 3:\n",
       "            cursor = 'move';\n",
       "            break;\n",
       "    }\n",
       "    fig.rubberband_canvas.style.cursor = cursor;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_message = function (fig, msg) {\n",
       "    fig.message.textContent = msg['message'];\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n",
       "    // Request the server to send over a new figure.\n",
       "    fig.send_draw_message();\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n",
       "    fig.image_mode = msg['mode'];\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n",
       "    for (var key in msg) {\n",
       "        if (!(key in fig.buttons)) {\n",
       "            continue;\n",
       "        }\n",
       "        fig.buttons[key].disabled = !msg[key];\n",
       "        fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n",
       "    if (msg['mode'] === 'PAN') {\n",
       "        fig.buttons['Pan'].classList.add('active');\n",
       "        fig.buttons['Zoom'].classList.remove('active');\n",
       "    } else if (msg['mode'] === 'ZOOM') {\n",
       "        fig.buttons['Pan'].classList.remove('active');\n",
       "        fig.buttons['Zoom'].classList.add('active');\n",
       "    } else {\n",
       "        fig.buttons['Pan'].classList.remove('active');\n",
       "        fig.buttons['Zoom'].classList.remove('active');\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.updated_canvas_event = function () {\n",
       "    // Called whenever the canvas gets updated.\n",
       "    this.send_message('ack', {});\n",
       "};\n",
       "\n",
       "// A function to construct a web socket function for onmessage handling.\n",
       "// Called in the figure constructor.\n",
       "mpl.figure.prototype._make_on_message_function = function (fig) {\n",
       "    return function socket_on_message(evt) {\n",
       "        if (evt.data instanceof Blob) {\n",
       "            /* FIXME: We get \"Resource interpreted as Image but\n",
       "             * transferred with MIME type text/plain:\" errors on\n",
       "             * Chrome.  But how to set the MIME type?  It doesn't seem\n",
       "             * to be part of the websocket stream */\n",
       "            evt.data.type = 'image/png';\n",
       "\n",
       "            /* Free the memory for the previous frames */\n",
       "            if (fig.imageObj.src) {\n",
       "                (window.URL || window.webkitURL).revokeObjectURL(\n",
       "                    fig.imageObj.src\n",
       "                );\n",
       "            }\n",
       "\n",
       "            fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n",
       "                evt.data\n",
       "            );\n",
       "            fig.updated_canvas_event();\n",
       "            fig.waiting = false;\n",
       "            return;\n",
       "        } else if (\n",
       "            typeof evt.data === 'string' &&\n",
       "            evt.data.slice(0, 21) === 'data:image/png;base64'\n",
       "        ) {\n",
       "            fig.imageObj.src = evt.data;\n",
       "            fig.updated_canvas_event();\n",
       "            fig.waiting = false;\n",
       "            return;\n",
       "        }\n",
       "\n",
       "        var msg = JSON.parse(evt.data);\n",
       "        var msg_type = msg['type'];\n",
       "\n",
       "        // Call the  \"handle_{type}\" callback, which takes\n",
       "        // the figure and JSON message as its only arguments.\n",
       "        try {\n",
       "            var callback = fig['handle_' + msg_type];\n",
       "        } catch (e) {\n",
       "            console.log(\n",
       "                \"No handler for the '\" + msg_type + \"' message type: \",\n",
       "                msg\n",
       "            );\n",
       "            return;\n",
       "        }\n",
       "\n",
       "        if (callback) {\n",
       "            try {\n",
       "                // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n",
       "                callback(fig, msg);\n",
       "            } catch (e) {\n",
       "                console.log(\n",
       "                    \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n",
       "                    e,\n",
       "                    e.stack,\n",
       "                    msg\n",
       "                );\n",
       "            }\n",
       "        }\n",
       "    };\n",
       "};\n",
       "\n",
       "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n",
       "mpl.findpos = function (e) {\n",
       "    //this section is from http://www.quirksmode.org/js/events_properties.html\n",
       "    var targ;\n",
       "    if (!e) {\n",
       "        e = window.event;\n",
       "    }\n",
       "    if (e.target) {\n",
       "        targ = e.target;\n",
       "    } else if (e.srcElement) {\n",
       "        targ = e.srcElement;\n",
       "    }\n",
       "    if (targ.nodeType === 3) {\n",
       "        // defeat Safari bug\n",
       "        targ = targ.parentNode;\n",
       "    }\n",
       "\n",
       "    // pageX,Y are the mouse positions relative to the document\n",
       "    var boundingRect = targ.getBoundingClientRect();\n",
       "    var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n",
       "    var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n",
       "\n",
       "    return { x: x, y: y };\n",
       "};\n",
       "\n",
       "/*\n",
       " * return a copy of an object with only non-object keys\n",
       " * we need this to avoid circular references\n",
       " * http://stackoverflow.com/a/24161582/3208463\n",
       " */\n",
       "function simpleKeys(original) {\n",
       "    return Object.keys(original).reduce(function (obj, key) {\n",
       "        if (typeof original[key] !== 'object') {\n",
       "            obj[key] = original[key];\n",
       "        }\n",
       "        return obj;\n",
       "    }, {});\n",
       "}\n",
       "\n",
       "mpl.figure.prototype.mouse_event = function (event, name) {\n",
       "    var canvas_pos = mpl.findpos(event);\n",
       "\n",
       "    if (name === 'button_press') {\n",
       "        this.canvas.focus();\n",
       "        this.canvas_div.focus();\n",
       "    }\n",
       "\n",
       "    var x = canvas_pos.x * mpl.ratio;\n",
       "    var y = canvas_pos.y * mpl.ratio;\n",
       "\n",
       "    this.send_message(name, {\n",
       "        x: x,\n",
       "        y: y,\n",
       "        button: event.button,\n",
       "        step: event.step,\n",
       "        guiEvent: simpleKeys(event),\n",
       "    });\n",
       "\n",
       "    /* This prevents the web browser from automatically changing to\n",
       "     * the text insertion cursor when the button is pressed.  We want\n",
       "     * to control all of the cursor setting manually through the\n",
       "     * 'cursor' event from matplotlib */\n",
       "    event.preventDefault();\n",
       "    return false;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n",
       "    // Handle any extra behaviour associated with a key event\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.key_event = function (event, name) {\n",
       "    // Prevent repeat events\n",
       "    if (name === 'key_press') {\n",
       "        if (event.which === this._key) {\n",
       "            return;\n",
       "        } else {\n",
       "            this._key = event.which;\n",
       "        }\n",
       "    }\n",
       "    if (name === 'key_release') {\n",
       "        this._key = null;\n",
       "    }\n",
       "\n",
       "    var value = '';\n",
       "    if (event.ctrlKey && event.which !== 17) {\n",
       "        value += 'ctrl+';\n",
       "    }\n",
       "    if (event.altKey && event.which !== 18) {\n",
       "        value += 'alt+';\n",
       "    }\n",
       "    if (event.shiftKey && event.which !== 16) {\n",
       "        value += 'shift+';\n",
       "    }\n",
       "\n",
       "    value += 'k';\n",
       "    value += event.which.toString();\n",
       "\n",
       "    this._key_event_extra(event, name);\n",
       "\n",
       "    this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n",
       "    return false;\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n",
       "    if (name === 'download') {\n",
       "        this.handle_save(this, null);\n",
       "    } else {\n",
       "        this.send_message('toolbar_button', { name: name });\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n",
       "    this.message.textContent = tooltip;\n",
       "};\n",
       "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n",
       "\n",
       "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n",
       "\n",
       "mpl.default_extension = \"png\";/* global mpl */\n",
       "\n",
       "var comm_websocket_adapter = function (comm) {\n",
       "    // Create a \"websocket\"-like object which calls the given IPython comm\n",
       "    // object with the appropriate methods. Currently this is a non binary\n",
       "    // socket, so there is still some room for performance tuning.\n",
       "    var ws = {};\n",
       "\n",
       "    ws.close = function () {\n",
       "        comm.close();\n",
       "    };\n",
       "    ws.send = function (m) {\n",
       "        //console.log('sending', m);\n",
       "        comm.send(m);\n",
       "    };\n",
       "    // Register the callback with on_msg.\n",
       "    comm.on_msg(function (msg) {\n",
       "        //console.log('receiving', msg['content']['data'], msg);\n",
       "        // Pass the mpl event to the overridden (by mpl) onmessage function.\n",
       "        ws.onmessage(msg['content']['data']);\n",
       "    });\n",
       "    return ws;\n",
       "};\n",
       "\n",
       "mpl.mpl_figure_comm = function (comm, msg) {\n",
       "    // This is the function which gets called when the mpl process\n",
       "    // starts-up an IPython Comm through the \"matplotlib\" channel.\n",
       "\n",
       "    var id = msg.content.data.id;\n",
       "    // Get hold of the div created by the display call when the Comm\n",
       "    // socket was opened in Python.\n",
       "    var element = document.getElementById(id);\n",
       "    var ws_proxy = comm_websocket_adapter(comm);\n",
       "\n",
       "    function ondownload(figure, _format) {\n",
       "        window.open(figure.canvas.toDataURL());\n",
       "    }\n",
       "\n",
       "    var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n",
       "\n",
       "    // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n",
       "    // web socket which is closed, not our websocket->open comm proxy.\n",
       "    ws_proxy.onopen();\n",
       "\n",
       "    fig.parent_element = element;\n",
       "    fig.cell_info = mpl.find_output_cell(\"<div id='\" + id + \"'></div>\");\n",
       "    if (!fig.cell_info) {\n",
       "        console.error('Failed to find cell for figure', id, fig);\n",
       "        return;\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_close = function (fig, msg) {\n",
       "    var width = fig.canvas.width / mpl.ratio;\n",
       "    fig.root.removeEventListener('remove', this._remove_fig_handler);\n",
       "\n",
       "    // Update the output cell to use the data from the current canvas.\n",
       "    fig.push_to_output();\n",
       "    var dataURL = fig.canvas.toDataURL();\n",
       "    // Re-enable the keyboard manager in IPython - without this line, in FF,\n",
       "    // the notebook keyboard shortcuts fail.\n",
       "    IPython.keyboard_manager.enable();\n",
       "    fig.parent_element.innerHTML =\n",
       "        '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
       "    fig.close_ws(fig, msg);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.close_ws = function (fig, msg) {\n",
       "    fig.send_message('closing', msg);\n",
       "    // fig.ws.close()\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n",
       "    // Turn the data on the canvas into data in the output cell.\n",
       "    var width = this.canvas.width / mpl.ratio;\n",
       "    var dataURL = this.canvas.toDataURL();\n",
       "    this.cell_info[1]['text/html'] =\n",
       "        '<img src=\"' + dataURL + '\" width=\"' + width + '\">';\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.updated_canvas_event = function () {\n",
       "    // Tell IPython that the notebook contents must change.\n",
       "    IPython.notebook.set_dirty(true);\n",
       "    this.send_message('ack', {});\n",
       "    var fig = this;\n",
       "    // Wait a second, then push the new image to the DOM so\n",
       "    // that it is saved nicely (might be nice to debounce this).\n",
       "    setTimeout(function () {\n",
       "        fig.push_to_output();\n",
       "    }, 1000);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._init_toolbar = function () {\n",
       "    var fig = this;\n",
       "\n",
       "    var toolbar = document.createElement('div');\n",
       "    toolbar.classList = 'btn-toolbar';\n",
       "    this.root.appendChild(toolbar);\n",
       "\n",
       "    function on_click_closure(name) {\n",
       "        return function (_event) {\n",
       "            return fig.toolbar_button_onclick(name);\n",
       "        };\n",
       "    }\n",
       "\n",
       "    function on_mouseover_closure(tooltip) {\n",
       "        return function (event) {\n",
       "            if (!event.currentTarget.disabled) {\n",
       "                return fig.toolbar_button_onmouseover(tooltip);\n",
       "            }\n",
       "        };\n",
       "    }\n",
       "\n",
       "    fig.buttons = {};\n",
       "    var buttonGroup = document.createElement('div');\n",
       "    buttonGroup.classList = 'btn-group';\n",
       "    var button;\n",
       "    for (var toolbar_ind in mpl.toolbar_items) {\n",
       "        var name = mpl.toolbar_items[toolbar_ind][0];\n",
       "        var tooltip = mpl.toolbar_items[toolbar_ind][1];\n",
       "        var image = mpl.toolbar_items[toolbar_ind][2];\n",
       "        var method_name = mpl.toolbar_items[toolbar_ind][3];\n",
       "\n",
       "        if (!name) {\n",
       "            /* Instead of a spacer, we start a new button group. */\n",
       "            if (buttonGroup.hasChildNodes()) {\n",
       "                toolbar.appendChild(buttonGroup);\n",
       "            }\n",
       "            buttonGroup = document.createElement('div');\n",
       "            buttonGroup.classList = 'btn-group';\n",
       "            continue;\n",
       "        }\n",
       "\n",
       "        button = fig.buttons[name] = document.createElement('button');\n",
       "        button.classList = 'btn btn-default';\n",
       "        button.href = '#';\n",
       "        button.title = name;\n",
       "        button.innerHTML = '<i class=\"fa ' + image + ' fa-lg\"></i>';\n",
       "        button.addEventListener('click', on_click_closure(method_name));\n",
       "        button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n",
       "        buttonGroup.appendChild(button);\n",
       "    }\n",
       "\n",
       "    if (buttonGroup.hasChildNodes()) {\n",
       "        toolbar.appendChild(buttonGroup);\n",
       "    }\n",
       "\n",
       "    // Add the status bar.\n",
       "    var status_bar = document.createElement('span');\n",
       "    status_bar.classList = 'mpl-message pull-right';\n",
       "    toolbar.appendChild(status_bar);\n",
       "    this.message = status_bar;\n",
       "\n",
       "    // Add the close button to the window.\n",
       "    var buttongrp = document.createElement('div');\n",
       "    buttongrp.classList = 'btn-group inline pull-right';\n",
       "    button = document.createElement('button');\n",
       "    button.classList = 'btn btn-mini btn-primary';\n",
       "    button.href = '#';\n",
       "    button.title = 'Stop Interaction';\n",
       "    button.innerHTML = '<i class=\"fa fa-power-off icon-remove icon-large\"></i>';\n",
       "    button.addEventListener('click', function (_evt) {\n",
       "        fig.handle_close(fig, {});\n",
       "    });\n",
       "    button.addEventListener(\n",
       "        'mouseover',\n",
       "        on_mouseover_closure('Stop Interaction')\n",
       "    );\n",
       "    buttongrp.appendChild(button);\n",
       "    var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n",
       "    titlebar.insertBefore(buttongrp, titlebar.firstChild);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._remove_fig_handler = function () {\n",
       "    this.close_ws(this, {});\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._root_extra_style = function (el) {\n",
       "    el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n",
       "    el.addEventListener('remove', this._remove_fig_handler);\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._canvas_extra_style = function (el) {\n",
       "    // this is important to make the div 'focusable\n",
       "    el.setAttribute('tabindex', 0);\n",
       "    // reach out to IPython and tell the keyboard manager to turn it's self\n",
       "    // off when our div gets focus\n",
       "\n",
       "    // location in version 3\n",
       "    if (IPython.notebook.keyboard_manager) {\n",
       "        IPython.notebook.keyboard_manager.register_events(el);\n",
       "    } else {\n",
       "        // location in version 2\n",
       "        IPython.keyboard_manager.register_events(el);\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype._key_event_extra = function (event, _name) {\n",
       "    var manager = IPython.notebook.keyboard_manager;\n",
       "    if (!manager) {\n",
       "        manager = IPython.keyboard_manager;\n",
       "    }\n",
       "\n",
       "    // Check for shift+enter\n",
       "    if (event.shiftKey && event.which === 13) {\n",
       "        this.canvas_div.blur();\n",
       "        // select the cell after this one\n",
       "        var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n",
       "        IPython.notebook.select(index + 1);\n",
       "    }\n",
       "};\n",
       "\n",
       "mpl.figure.prototype.handle_save = function (fig, _msg) {\n",
       "    fig.ondownload(fig, null);\n",
       "};\n",
       "\n",
       "mpl.find_output_cell = function (html_output) {\n",
       "    // Return the cell and output element which can be found *uniquely* in the notebook.\n",
       "    // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n",
       "    // IPython event is triggered only after the cells have been serialised, which for\n",
       "    // our purposes (turning an active figure into a static one), is too late.\n",
       "    var cells = IPython.notebook.get_cells();\n",
       "    var ncells = cells.length;\n",
       "    for (var i = 0; i < ncells; i++) {\n",
       "        var cell = cells[i];\n",
       "        if (cell.cell_type === 'code') {\n",
       "            for (var j = 0; j < cell.output_area.outputs.length; j++) {\n",
       "                var data = cell.output_area.outputs[j];\n",
       "                if (data.data) {\n",
       "                    // IPython >= 3 moved mimebundle to data attribute of output\n",
       "                    data = data.data;\n",
       "                }\n",
       "                if (data['text/html'] === html_output) {\n",
       "                    return [cell, data, j];\n",
       "                }\n",
       "            }\n",
       "        }\n",
       "    }\n",
       "};\n",
       "\n",
       "// Register the function which deals with the matplotlib target/channel.\n",
       "// The kernel may be null if the page has been refreshed.\n",
       "if (IPython.notebook.kernel !== null) {\n",
       "    IPython.notebook.kernel.comm_manager.register_target(\n",
       "        'matplotlib',\n",
       "        mpl.mpl_figure_comm\n",
       "    );\n",
       "}\n"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<img src=\"\" width=\"1000\">"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib notebook\n",
    "\n",
    "f, ax = plt.subplots(2, 2, figsize = (10, 10))\n",
    "\n",
    "\n",
    "ax[0][0].plot(x_test, uv_pred[0][:, 0:1], label = \"pred\")\n",
    "ax[0][0].plot(x_test, u_true[:, 0], label = \"true\")\n",
    "ax[0][0].set_title(\"u, t = {:.4f}\".format(t_test[0, 0]))\n",
    "\n",
    "ax[0][1].plot(x_test, uv_pred[0][:, 1:2], label = \"pred\")\n",
    "ax[0][1].plot(x_test, v_true[:, 0], label = \"true\")\n",
    "ax[0][1].set_title(\"v, t = {:.4f}\".format(t_test[0, 0]))\n",
    "\n",
    "ax[1][0].plot(x_test, uv_pred[-1][:, 0:1], label = \"pred\")\n",
    "ax[1][0].plot(x_test, u_true[:, -1], label = \"true\")\n",
    "ax[1][0].set_title(\"u, t = {:.4f}\".format(t_test[-1, 0]))\n",
    "\n",
    "ax[1][1].plot(x_test, uv_pred[-1][:, 1:2], label = \"pred\")\n",
    "ax[1][1].plot(x_test, v_true[:, -1], label = \"true\")\n",
    "ax[1][1].set_title(\"v, t = {:.4f}\".format(t_test[-1, 0]))\n",
    "\n",
    "for i in range(2):\n",
    "\tfor j in range(2):\n",
    "\t\tax[i][j].legend()\n",
    "\t\tax[i][j].grid()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
